Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

feat(bots): name the bot, split AI crawlers from AI agents (#1147)

Goldflag479297552f0c
diff --git a/client/src/api/analytics/endpoints/bots.ts b/client/src/api/analytics/endpoints/bots.ts
index 124947059..524c48936 100644
--- a/client/src/api/analytics/endpoints/bots.ts
+++ b/client/src/api/analytics/endpoints/bots.ts
@@ -3,17 +3,62 @@ import { CommonApiParams, PaginationParams } from "./types";
export type BotLayerKey = "ua_pattern" | "header_heuristics" | "client_signals" | "bot_asn" | "rate_anomaly";
-export type BotDimensionKey = FilterParameter | "asn_org" | "bot_category" | "matched_ua_pattern";
+export type BotDimensionKey =
+ | FilterParameter
+ | "asn_org"
+ | "asn_provider"
+ | "bot_category"
+ | "bot_name"
+ | "bot_operator"
+ | "bot_purpose"
+ | "matched_ua_pattern";
+
+/**
+ * What the operator does with the fetch. `bot_category` groups by family; this
+ * separates the three things that all read as "ai" β€” a crawler building a
+ * training corpus, an answer engine indexing for retrieval, and a fetch a
+ * person asked for a moment ago.
+ */
+export type BotPurpose =
+ | "ai_training"
+ | "ai_search"
+ | "ai_agent"
+ | "search"
+ | "social_preview"
+ | "seo"
+ | "monitoring"
+ | "security"
+ | "scripted"
+ | "headless";
export type GetBotOverviewResponse = Record<BotLayerKey, number> & {
bot_requests: number;
total_events: number;
bot_percentage: number;
+ ai_requests: number;
+ ai_agent_requests: number;
+ ai_crawler_requests: number;
+};
+
+/** One AI operator: how much of the site it read, and what it sent back. */
+export type BotAiSummaryRow = {
+ operator: string;
+ crawls: number;
+ training_crawls: number;
+ search_crawls: number;
+ agent_requests: number;
+ referrals: number;
+ crawls_per_referral: number;
};
+export type GetBotAiSummaryResponse = BotAiSummaryRow[];
+
export type BotTimeSeriesPoint = {
time: string;
bot_requests: number;
+ /** Both returned on every bucket so the AI chart needs no second request. */
+ ai_agent_requests: number;
+ ai_crawler_requests: number;
};
export type GetBotTimeSeriesResponse = BotTimeSeriesPoint[];
diff --git a/client/src/api/analytics/endpoints/index.ts b/client/src/api/analytics/endpoints/index.ts
index 1b7717fad..90febaf5f 100644
--- a/client/src/api/analytics/endpoints/index.ts
+++ b/client/src/api/analytics/endpoints/index.ts
@@ -126,13 +126,16 @@ export type {
// Bots types
export type {
+ BotAiSummaryRow,
BotDimensionKey,
BotDimensionItem,
BotDimensionParams,
BotLayerKey,
BotOverviewParams,
+ BotPurpose,
BotTimeSeriesParams,
BotTimeSeriesPoint,
+ GetBotAiSummaryResponse,
GetBotOverviewResponse,
GetBotTimeSeriesResponse,
PaginatedBotDimensionResponse,
diff --git a/client/src/api/analytics/hooks/bots/useGetBotAiSummary.ts b/client/src/api/analytics/hooks/bots/useGetBotAiSummary.ts
new file mode 100644
index 000000000..18253da60
--- /dev/null
+++ b/client/src/api/analytics/hooks/bots/useGetBotAiSummary.ts
@@ -0,0 +1,24 @@
+import { useStore } from "../../../../lib/store";
+import { type GetBotAiSummaryResponse } from "../../endpoints";
+import { useAnalyticsQuery } from "../../useAnalyticsQuery";
+import { BOT_AVAILABLE_FILTERS } from "./constants";
+
+/**
+ * Crawls and referrals per AI operator, side by side.
+ *
+ * Deliberately does not take the layer selection: layers describe how a bot was
+ * caught, and every row here was caught the same way (its user agent named it).
+ */
+export function useGetBotAiSummary({ site }: { site?: number | string }) {
+ const filters = useStore(state => state.filters);
+ const botFilters = filters.filter(filter => BOT_AVAILABLE_FILTERS.includes(filter.parameter));
+
+ return useAnalyticsQuery<GetBotAiSummaryResponse>({
+ key: "bot-ai-summary",
+ path: "bots/ai-summary",
+ site,
+ // Only bot-relevant filters go on the wire; when none apply, send no filters.
+ useFilters: botFilters.length > 0,
+ customFilters: botFilters,
+ });
+}
diff --git a/client/src/api/analytics/hooks/bots/useGetBotDimension.ts b/client/src/api/analytics/hooks/bots/useGetBotDimension.ts
index 7684ade7b..b67e05eb7 100644
--- a/client/src/api/analytics/hooks/bots/useGetBotDimension.ts
+++ b/client/src/api/analytics/hooks/bots/useGetBotDimension.ts
@@ -7,11 +7,14 @@ import { BOT_AVAILABLE_FILTERS } from "./constants";
export function useGetBotDimension({
site,
dimension,
+ purpose,
limit = 100,
page = 1,
}: {
site?: number | string;
dimension: BotDimensionKey;
+ /** A single purpose, or "ai" / "ai_crawler" for the grouped families. */
+ purpose?: string;
limit?: number;
page?: number;
}) {
@@ -26,6 +29,6 @@ export function useGetBotDimension({
// Only bot-relevant filters go on the wire; when none apply, send no filters.
useFilters: botFilters.length > 0,
customFilters: botFilters,
- params: { dimension, limit, page, layer: selectedLayer || undefined },
+ params: { dimension, limit, page, purpose, layer: selectedLayer || undefined },
});
}
diff --git a/client/src/api/analytics/hooks/bots/useGetBotTimeSeries.ts b/client/src/api/analytics/hooks/bots/useGetBotTimeSeries.ts
index 823f9c341..491fa3c6d 100644
--- a/client/src/api/analytics/hooks/bots/useGetBotTimeSeries.ts
+++ b/client/src/api/analytics/hooks/bots/useGetBotTimeSeries.ts
@@ -9,10 +9,13 @@ import { BOT_AVAILABLE_FILTERS } from "./constants";
export function useGetBotTimeSeries({
site,
bucket,
+ purpose,
props,
}: {
site: number | string;
bucket?: TimeBucket;
+ /** A single purpose, or "ai" / "ai_crawler" for the grouped families. */
+ purpose?: string;
props?: Partial<UseQueryOptions<GetBotTimeSeriesResponse, Error>>;
}): UseQueryResult<GetBotTimeSeriesResponse> {
const filters = useStore(state => state.filters);
@@ -27,7 +30,7 @@ export function useGetBotTimeSeries({
// Only bot-relevant filters go on the wire; when none apply, send no filters.
useFilters: botFilters.length > 0,
customFilters: botFilters,
- params: { bucket: bucket || storeBucket, layer: selectedLayer || undefined },
+ params: { bucket: bucket || storeBucket, purpose, layer: selectedLayer || undefined },
props,
});
}
diff --git a/client/src/app/[site]/api-playground/utils/endpointConfig.ts b/client/src/app/[site]/api-playground/utils/endpointConfig.ts
index 1fed8241d..2cbcde38a 100644
--- a/client/src/app/[site]/api-playground/utils/endpointConfig.ts
+++ b/client/src/app/[site]/api-playground/utils/endpointConfig.ts
@@ -422,7 +422,14 @@ export const endpointCategories: EndpointCategory[] = [
name: "Get Bot Time Series",
description: "Returns bot request counts over time",
hasCommonParams: true,
- specificParams: ["bucket", "layer"],
+ specificParams: ["bucket", "layer", "purpose"],
+ },
+ {
+ method: "GET",
+ path: "/sites/:site/bots/ai-summary",
+ name: "Get AI Summary",
+ description: "Returns crawls and referrals per AI operator, side by side",
+ hasCommonParams: true,
},
{
method: "GET",
@@ -431,7 +438,7 @@ export const endpointCategories: EndpointCategory[] = [
description: "Returns bot requests broken down by a dimension",
hasCommonParams: true,
requiredParams: ["dimension"],
- specificParams: ["dimension", "limit", "page", "layer"],
+ specificParams: ["dimension", "limit", "page", "layer", "purpose"],
parameterMetadata: {
dimension: {
label: "Dimension",
@@ -450,7 +457,11 @@ export const endpointCategories: EndpointCategory[] = [
"pathname",
"dimensions",
"asn_org",
+ "asn_provider",
"bot_category",
+ "bot_name",
+ "bot_operator",
+ "bot_purpose",
"matched_ua_pattern",
],
},
@@ -896,6 +907,25 @@ export const parameterMetadata: Record<string, ParameterMetadata> = {
type: "select",
options: ["ua_pattern", "header_heuristics", "client_signals", "bot_asn", "rate_anomaly"],
},
+ purpose: {
+ label: "Purpose",
+ type: "select",
+ // "ai" and "ai_crawler" are groups; the rest are the stored values.
+ options: [
+ "ai",
+ "ai_crawler",
+ "ai_training",
+ "ai_search",
+ "ai_agent",
+ "search",
+ "social_preview",
+ "seo",
+ "monitoring",
+ "security",
+ "scripted",
+ "headless",
+ ],
+ },
mode: {
label: "Mode",
type: "select",
diff --git a/client/src/app/[site]/bots/botsStore.ts b/client/src/app/[site]/bots/botsStore.ts
index 16e5e117c..9e12e7ade 100644
--- a/client/src/app/[site]/bots/botsStore.ts
+++ b/client/src/app/[site]/bots/botsStore.ts
@@ -1,12 +1,22 @@
import { create } from "zustand";
import { type BotLayerKey } from "../../../api/analytics/endpoints";
+/** Which question the page is answering. */
+export type BotsLens = "ai" | "all";
+
type BotsStore = {
+ lens: BotsLens;
+ setLens: (lens: BotsLens) => void;
selectedLayer: BotLayerKey | null;
setSelectedLayer: (layer: BotLayerKey | null) => void;
};
export const useBotsStore = create<BotsStore>(set => ({
+ lens: "ai",
+ // Layer selection belongs to the all-bots view. Carrying it into the AI lens
+ // would silently narrow every AI number to one detection layer with nothing
+ // on screen saying so.
+ setLens: lens => set(lens === "ai" ? { lens, selectedLayer: null } : { lens }),
selectedLayer: null,
setSelectedLayer: layer => set({ selectedLayer: layer }),
}));
diff --git a/client/src/app/[site]/bots/components/BotSection.tsx b/client/src/app/[site]/bots/components/BotSection.tsx
index de4ea6d6c..3be5e263b 100644
--- a/client/src/app/[site]/bots/components/BotSection.tsx
+++ b/client/src/app/[site]/bots/components/BotSection.tsx
@@ -133,11 +133,12 @@ function BotRows({
);
}
-function useBotSectionData(dimension: BotDimensionKey) {
+function useBotSectionData(dimension: BotDimensionKey, purpose?: string) {
const { site } = useStore();
const { data, isLoading, isFetching, error, refetch } = useGetBotDimension({
site,
dimension,
+ purpose,
limit: 100,
page: 1,
});
@@ -170,6 +171,8 @@ export type BotSectionBaseProps = {
getValue: (item: BotDimensionItem) => string;
getLink?: (item: BotDimensionItem) => string | undefined;
filterable?: boolean;
+ /** Narrows the section to one bot purpose, or to "ai" / "ai_crawler". */
+ purpose?: string;
};
type BotSectionProps = BotSectionBaseProps & {
@@ -185,8 +188,9 @@ export function BotSectionDialogBody({
getValue,
getLink,
filterable = true,
+ purpose,
}: BotSectionBaseProps) {
- const { items, ratio, isLoading, error, refetch } = useBotSectionData(dimension);
+ const { items, ratio, isLoading, error, refetch } = useBotSectionData(dimension, purpose);
const filterParameter = filterable ? (dimension as FilterParameter) : undefined;
if (isLoading) {
@@ -230,9 +234,10 @@ export function BotSection({
expanded,
close,
filterable = true,
+ purpose,
renderDialog = true,
}: BotSectionProps) {
- const { items, ratio, isLoading, isFetching, error, refetch } = useBotSectionData(dimension);
+ const { items, ratio, isLoading, isFetching, error, refetch } = useBotSectionData(dimension, purpose);
const filterParameter = filterable ? (dimension as FilterParameter) : undefined;
const content = (
diff --git a/client/src/app/[site]/bots/components/ai/AiBots.tsx b/client/src/app/[site]/bots/components/ai/AiBots.tsx
new file mode 100644
index 000000000..15c3f57b5
--- /dev/null
+++ b/client/src/app/[site]/bots/components/ai/AiBots.tsx
@@ -0,0 +1,55 @@
+"use client";
+
+import { BotSectionTabs, type BotSectionTab } from "../BotSectionTabs";
+import { formatBotPurpose } from "./aiLabels";
+
+type Tab = "bots" | "purposes" | "operators";
+
+/** Who is reading the site, at three levels of resolution. */
+export function AiBots() {
+ const tabs: BotSectionTab<Tab>[] = [
+ {
+ value: "bots",
+ label: "Bots",
+ section: {
+ dimension: "bot_name",
+ title: "AI bots",
+ purpose: "ai",
+ getValue: item => item.value,
+ getKey: item => item.value || "unnamed",
+ // A bot whose user agent matched only a generic pattern has no
+ // published name. Empty rows also come from before identity shipped.
+ getLabel: item => item.value || "Unnamed",
+ filterable: false,
+ },
+ },
+ {
+ value: "purposes",
+ label: "Purpose",
+ section: {
+ dimension: "bot_purpose",
+ title: "What they came for",
+ purpose: "ai",
+ getValue: item => item.value,
+ getKey: item => item.value || "unclassified",
+ getLabel: item => formatBotPurpose(item.value),
+ filterable: false,
+ },
+ },
+ {
+ value: "operators",
+ label: "Operators",
+ section: {
+ dimension: "bot_operator",
+ title: "AI operators",
+ purpose: "ai",
+ getValue: item => item.value,
+ getKey: item => item.value || "unknown",
+ getLabel: item => item.value || "Unknown",
+ filterable: false,
+ },
+ },
+ ];
+
+ return <BotSectionTabs defaultValue="bots" tabs={tabs} />;
+}
diff --git a/client/src/app/[site]/bots/components/ai/AiChart.tsx b/client/src/app/[site]/bots/components/ai/AiChart.tsx
new file mode 100644
index 000000000..91d68908b
--- /dev/null
+++ b/client/src/app/[site]/bots/components/ai/AiChart.tsx
@@ -0,0 +1,142 @@
+"use client";
+
+import { DateTime } from "luxon";
+import { useMemo } from "react";
+import { useGetBotTimeSeries } from "../../../../../api/analytics/hooks/bots/useGetBotTimeSeries";
+import { BucketSelection } from "../../../../../components/BucketSelection";
+import { ChartTooltip } from "../../../../../components/charts/ChartTooltip";
+import { TimeSeriesChart } from "../../../../../components/charts/TimeSeriesChart";
+import type { TimeSeriesChartPoint } from "../../../../../components/charts/TimeSeriesChart";
+import { getChartTimeBounds } from "../../../../../components/charts/timeSeriesChartUtils";
+import { Card, CardContent, CardLoader } from "../../../../../components/ui/card";
+import { Skeleton } from "../../../../../components/ui/skeleton";
+import { formatChartDateTime } from "../../../../../lib/dateTimeUtils";
+import { getTimezone, useStore } from "../../../../../lib/store";
+import { AI_AGENT_COLOR, AI_CRAWLER_COLOR } from "./aiLabels";
+
+type AiPoint = TimeSeriesChartPoint & {
+ currentTime: DateTime;
+ agents: number;
+ crawlers: number;
+};
+
+function LegendSwatch({ color, label }: { color: string; label: string }) {
+ return (
+ <div className="flex items-center gap-1.5">
+ <div className="w-2.5 h-2.5 rounded-[3px]" style={{ backgroundColor: color }} />
+ <span className="text-xs text-neutral-700 dark:text-neutral-200">{label}</span>
+ </div>
+ );
+}
+
+export function AiChart() {
+ const { site, bucket, time } = useStore();
+ const timezone = getTimezone();
+
+ const { data, isLoading, isFetching } = useGetBotTimeSeries({ site, purpose: "ai" });
+
+ const { agentPoints, crawlerPoints, chartMin, chartMax, max } = useMemo(() => {
+ const { min: boundsMin, max: boundsMax } = getChartTimeBounds(time, bucket, timezone);
+
+ const now = DateTime.now();
+ const lowerBoundMs = boundsMin?.getTime();
+ const upperBoundMs = (boundsMax ?? now.toJSDate()).getTime();
+ const agents: AiPoint[] = [];
+ const crawlers: AiPoint[] = [];
+
+ data?.forEach(item => {
+ const timestamp = DateTime.fromSQL(item.time, { zone: timezone }).toUTC();
+ if (timestamp > now) return;
+ const timestampMs = timestamp.toMillis();
+ if (lowerBoundMs !== undefined && timestampMs < lowerBoundMs) return;
+ if (timestampMs > upperBoundMs) return;
+
+ const shared = {
+ x: timestamp.toJSDate(),
+ currentTime: timestamp,
+ agents: item.ai_agent_requests,
+ crawlers: item.ai_crawler_requests,
+ };
+ agents.push({ ...shared, y: item.ai_agent_requests });
+ crawlers.push({ ...shared, y: item.ai_crawler_requests });
+ });
+
+ const dataMin = agents.length ? agents[0].x : undefined;
+ const dataMax = agents.length ? agents[agents.length - 1].x : undefined;
+
+ return {
+ agentPoints: agents,
+ crawlerPoints: crawlers,
+ chartMin: dataMin ?? boundsMin,
+ chartMax: dataMax ?? boundsMax ?? now.toJSDate(),
+ max: agents.reduce((largest, point) => Math.max(largest, point.agents, point.crawlers), 0),
+ };
+ }, [bucket, data, time, timezone]);
+
+ const hasData = agentPoints.some(point => point.agents > 0 || point.crawlers > 0);
+
+ return (
+ <Card className="overflow-visible">
+ {isFetching && (
+ <div className="absolute inset-x-0 top-0 h-4 overflow-hidden rounded-t-lg pointer-events-none">
+ <CardLoader />
+ </div>
+ )}
+ <CardContent className="p-2 md:p-4 py-3 w-full">
+ <div className="flex items-center justify-between px-2 md:px-0">
+ <div className="flex items-center gap-4">
+ <LegendSwatch color={AI_AGENT_COLOR} label="Agents" />
+ <LegendSwatch color={AI_CRAWLER_COLOR} label="Crawlers" />
+ </div>
+ <BucketSelection />
+ </div>
+ {isLoading ? (
+ <Skeleton className="w-full h-[300px] rounded-md mt-3" />
+ ) : !hasData ? (
+ <div className="h-[300px] w-full flex items-center justify-center text-neutral-500">
+ <div className="text-center">
+ <p className="text-lg font-medium">No AI traffic in this period</p>
+ <p className="text-sm">Try a wider date range, or check the coverage note below.</p>
+ </div>
+ </div>
+ ) : (
+ <div className="h-[300px] w-full">
+ <TimeSeriesChart
+ current={agentPoints}
+ max={max}
+ chartMin={chartMin}
+ chartMax={chartMax}
+ currentColor={AI_AGENT_COLOR}
+ series={[
+ { id: "agents", data: agentPoints, color: AI_AGENT_COLOR },
+ { id: "crawlers", data: crawlerPoints, color: AI_CRAWLER_COLOR },
+ ]}
+ yTickFormat={value => Number(value).toLocaleString()}
+ renderTooltip={({ point, bucket: tooltipBucket }) => (
+ <ChartTooltip>
+ <div className="p-3 min-w-[180px]">
+ <div className="mb-2">{formatChartDateTime(point.currentTime, tooltipBucket)}</div>
+ <div className="flex justify-between items-center gap-4">
+ <div className="flex items-center gap-2">
+ <div className="w-1 h-3 rounded-[3px]" style={{ backgroundColor: AI_AGENT_COLOR }} />
+ <span>Agents</span>
+ </div>
+ <span className="font-medium">{point.agents.toLocaleString()}</span>
+ </div>
+ <div className="flex justify-between items-center gap-4 mt-1">
+ <div className="flex items-center gap-2">
+ <div className="w-1 h-3 rounded-[3px]" style={{ backgroundColor: AI_CRAWLER_COLOR }} />
+ <span>Crawlers</span>
+ </div>
+ <span className="font-medium">{point.crawlers.toLocaleString()}</span>
+ </div>
+ </div>
+ </ChartTooltip>
+ )}
+ />
+ </div>
+ )}
+ </CardContent>
+ </Card>
+ );
+}
diff --git a/client/src/app/[site]/bots/components/ai/AiCoverageNote.tsx b/client/src/app/[site]/bots/components/ai/AiCoverageNote.tsx
new file mode 100644
index 000000000..f622a75fb
--- /dev/null
+++ b/client/src/app/[site]/bots/components/ai/AiCoverageNote.tsx
@@ -0,0 +1,22 @@
+"use client";
+
+import { Info } from "lucide-react";
+
+/**
+ * The dashboard is honest about its vantage point or it is not trustworthy.
+ * Rybbit only sees a request if the page ran its script, so a crawler that
+ * fetches HTML and executes nothing never appears here at all β€” and without
+ * saying so, a low AI number reads as "nobody is crawling me" when it may mean
+ * "the ones crawling you are invisible from the browser".
+ */
+export function AiCoverageNote() {
+ return (
+ <div className="flex items-start gap-2 px-1 text-xs text-neutral-600 dark:text-neutral-400 leading-relaxed">
+ <Info className="h-3.5 w-3.5 mt-0.5 shrink-0" />
+ <p>
+ These counts cover bots that execute JavaScript. Crawlers that fetch your HTML and run nothing β€” which includes
+ most training crawlers β€” never reach Rybbit unless your server reports them through the tracking API.
+ </p>
+ </div>
+ );
+}
diff --git a/client/src/app/[site]/bots/components/ai/AiOperatorTable.tsx b/client/src/app/[site]/bots/components/ai/AiOperatorTable.tsx
new file mode 100644
index 000000000..9fc941211
--- /dev/null
+++ b/client/src/app/[site]/bots/components/ai/AiOperatorTable.tsx
@@ -0,0 +1,181 @@
+"use client";
+
+import {
+ createColumnHelper,
+ flexRender,
+ getCoreRowModel,
+ getSortedRowModel,
+ SortingState,
+ useReactTable,
+} from "@tanstack/react-table";
+import { Info } from "lucide-react";
+import { useMemo, useState } from "react";
+import { type BotAiSummaryRow } from "../../../../../api/analytics/endpoints";
+import { useGetBotAiSummary } from "../../../../../api/analytics/hooks/bots/useGetBotAiSummary";
+import { ErrorState } from "../../../../../components/ErrorState";
+import { Card, CardContent, CardLoader } from "../../../../../components/ui/card";
+import { ScrollArea } from "../../../../../components/ui/scroll-area";
+import { Skeleton } from "../../../../../components/ui/skeleton";
+import { TableSortIndicator } from "../../../../../components/ui/table";
+import {
+ Tooltip,
+ TooltipContent,
+ TooltipTrigger,
+} from "../../../../../components/ui/tooltip";
+import { useStore } from "../../../../../lib/store";
+import { cn } from "../../../../../lib/utils";
+
+const columnHelper = createColumnHelper<BotAiSummaryRow>();
+
+const number = (value: number) => Number(value ?? 0).toLocaleString();
+
+/**
+ * The exchange rate, and the only number on the page that is an opinion rather
+ * than a count. Reads as "N pages read per visit sent back"; an operator that
+ * sent nobody back has no rate at all, which is worth showing as an em dash
+ * rather than as a very large number or a zero.
+ */
+function RatioCell({ row }: { row: BotAiSummaryRow }) {
+ // A rate needs both halves. An operator that sent nobody back has no rate,
+ // and one that read nothing has no rate either β€” "0:1" would read as a
+ // spectacular exchange rate rather than as an absence.
+ if (!row.referrals || !row.crawls) {
+ return <span className="text-neutral-500">β€”</span>;
+ }
+ return <span>{Number(row.crawls_per_referral ?? 0).toLocaleString()}:1</span>;
+}
+
+export function AiOperatorTable() {
+ const { site } = useStore();
+ const { data, isLoading, isFetching, error, refetch } = useGetBotAiSummary({ site });
+ const [sorting, setSorting] = useState<SortingState>([{ id: "crawls", desc: true }]);
+
+ const rows = useMemo(() => data ?? [], [data]);
+
+ const columns = useMemo(
+ () => [
+ columnHelper.accessor("operator", {
+ header: "Operator",
+ cell: info => <span className="font-medium">{info.getValue()}</span>,
+ }),
+ columnHelper.accessor("agent_requests", {
+ header: "Agent",
+ cell: info => number(info.getValue()),
+ }),
+ columnHelper.accessor("search_crawls", {
+ header: "Answer engine",
+ cell: info => number(info.getValue()),
+ }),
+ columnHelper.accessor("training_crawls", {
+ header: "Training",
+ cell: info => number(info.getValue()),
+ }),
+ columnHelper.accessor("crawls", {
+ header: "Total reads",
+ cell: info => <span className="font-medium">{number(info.getValue())}</span>,
+ }),
+ columnHelper.accessor("referrals", {
+ header: "Visits back",
+ cell: info => number(info.getValue()),
+ }),
+ columnHelper.accessor("crawls_per_referral", {
+ header: "Reads per visit",
+ cell: info => <RatioCell row={info.row.original} />,
+ }),
+ ],
+ []
+ );
+
+ const table = useReactTable({
+ data: rows,
+ columns,
+ state: { sorting },
+ onSortingChange: setSorting,
+ getCoreRowModel: getCoreRowModel(),
+ getSortedRowModel: getSortedRowModel(),
+ });
+
+ return (
+ <Card>
+ {isFetching && !isLoading && <CardLoader />}
+ <CardContent className="p-2 md:p-4">
+ <div className="flex items-center gap-1.5 mb-3">
+ <h3 className="text-sm font-medium">AI operators</h3>
+ <Tooltip>
+ <TooltipTrigger asChild>
+ <span className="inline-flex cursor-help">
+ <Info className="h-3 w-3 text-neutral-600 dark:text-neutral-300" />
+ </span>
+ </TooltipTrigger>
+ <TooltipContent side="top" className="max-w-sm p-3">
+ <p className="text-sm text-neutral-600 dark:text-neutral-200 leading-relaxed">
+ How much each company read, and how many people it sent back. Reads come from requests whose user agent
+ names the operator; visits back are sessions that arrived from that company&apos;s product.
+ </p>
+ </TooltipContent>
+ </Tooltip>
+ </div>
+
+ {error ? (
+ <ErrorState title="Failed to load data" message={error.message} refetch={refetch} />
+ ) : isLoading ? (
+ <div className="space-y-2">
+ {Array.from({ length: 5 }).map((_, index) => (
+ <Skeleton key={index} className="h-8 w-full rounded-md" />
+ ))}
+ </div>
+ ) : rows.length === 0 ? (
+ <div className="text-neutral-600 dark:text-neutral-300 w-full text-center py-10 flex flex-row gap-2 items-center justify-center">
+ <Info className="w-5 h-5" />
+ No AI traffic in this period
+ </div>
+ ) : (
+ <ScrollArea className="max-h-[360px] overflow-x-auto">
+ <table className="w-full text-sm">
+ <thead>
+ {table.getHeaderGroups().map(headerGroup => (
+ <tr key={headerGroup.id} className="border-b border-neutral-200 dark:border-neutral-800">
+ {headerGroup.headers.map((header, index) => (
+ <th
+ key={header.id}
+ onClick={header.column.getToggleSortingHandler()}
+ className={cn(
+ "py-2 px-2 text-xs font-medium text-muted-foreground cursor-pointer select-none whitespace-nowrap",
+ index === 0 ? "text-left" : "text-right"
+ )}
+ >
+ <div
+ className={cn("flex items-center gap-1", index === 0 ? "justify-start" : "justify-end")}
+ >
+ {flexRender(header.column.columnDef.header, header.getContext())}
+ <TableSortIndicator sortDirection={header.column.getIsSorted()} />
+ </div>
+ </th>
+ ))}
+ </tr>
+ ))}
+ </thead>
+ <tbody>
+ {table.getRowModel().rows.map(row => (
+ <tr
+ key={row.id}
+ className="border-b border-neutral-100 dark:border-neutral-850 last:border-b-0 hover:bg-neutral-50 dark:hover:bg-neutral-900"
+ >
+ {row.getVisibleCells().map((cell, index) => (
+ <td
+ key={cell.id}
+ className={cn("py-2 px-2 whitespace-nowrap", index === 0 ? "text-left" : "text-right")}
+ >
+ {flexRender(cell.column.columnDef.cell, cell.getContext())}
+ </td>
+ ))}
+ </tr>
+ ))}
+ </tbody>
+ </table>
+ </ScrollArea>
+ )}
+ </CardContent>
+ </Card>
+ );
+}
diff --git a/client/src/app/[site]/bots/components/ai/AiOverview.tsx b/client/src/app/[site]/bots/components/ai/AiOverview.tsx
new file mode 100644
index 000000000..30ef856dd
--- /dev/null
+++ b/client/src/app/[site]/bots/components/ai/AiOverview.tsx
@@ -0,0 +1,97 @@
+"use client";
+
+import NumberFlow from "@number-flow/react";
+import { useMemo } from "react";
+import { useGetBotAiSummary } from "../../../../../api/analytics/hooks/bots/useGetBotAiSummary";
+import { useGetBotOverview } from "../../../../../api/analytics/hooks/bots/useGetBotOverview";
+import { Card, CardContent, CardLoader } from "../../../../../components/ui/card";
+import { Skeleton } from "../../../../../components/ui/skeleton";
+import {
+ Tooltip,
+ TooltipContent,
+ TooltipTrigger,
+} from "../../../../../components/ui/tooltip";
+import { useStore } from "../../../../../lib/store";
+import { HelpCircle } from "lucide-react";
+
+function AiStat({
+ label,
+ value,
+ hint,
+ isLoading,
+}: {
+ label: string;
+ value: number;
+ hint: string;
+ isLoading: boolean;
+}) {
+ return (
+ <div className="border-r border-b border-neutral-100 dark:border-neutral-800 last:border-r-0 p-3">
+ <div className="flex items-center gap-1.5 text-xs font-medium text-muted-foreground">
+ <span>{label}</span>
+ <Tooltip>
+ <TooltipTrigger asChild>
+ <span className="inline-flex cursor-help">
+ <HelpCircle className="h-3 w-3 text-neutral-600 dark:text-neutral-300 hover:text-neutral-900 dark:hover:text-neutral-100" />
+ </span>
+ </TooltipTrigger>
+ <TooltipContent side="top" className="max-w-xs p-3">
+ <p className="text-sm text-neutral-600 dark:text-neutral-200 leading-relaxed">{hint}</p>
+ </TooltipContent>
+ </Tooltip>
+ </div>
+ <div className="text-2xl font-medium mt-1">
+ {isLoading ? (
+ <Skeleton className="h-8 w-20 rounded-md" />
+ ) : (
+ <NumberFlow respectMotionPreference={false} value={value} format={{ notation: "compact" }} />
+ )}
+ </div>
+ </div>
+ );
+}
+
+export function AiOverview() {
+ const { site } = useStore();
+ const { data: overview, isLoading, isFetching } = useGetBotOverview({ site });
+ const { data: summary, isLoading: isSummaryLoading } = useGetBotAiSummary({ site });
+
+ const referrals = useMemo(
+ () => (summary ?? []).reduce((total, row) => total + Number(row.referrals ?? 0), 0),
+ [summary]
+ );
+
+ return (
+ <Card>
+ {isFetching && <CardLoader />}
+ <CardContent className="p-0">
+ <div className="grid grid-cols-2 md:grid-cols-4">
+ <AiStat
+ label="AI requests"
+ value={overview?.ai_requests ?? 0}
+ hint="Every request from a named AI system: training crawlers, answer-engine indexers, and agents fetching on a person's behalf."
+ isLoading={isLoading}
+ />
+ <AiStat
+ label="AI agents"
+ value={overview?.ai_agent_requests ?? 0}
+ hint="A person asked an assistant to open a page and it fetched it there and then β€” ChatGPT-User, Claude-User, Perplexity-User and friends. Closer to a visit than to a crawl."
+ isLoading={isLoading}
+ />
+ <AiStat
+ label="AI crawlers"
+ value={overview?.ai_crawler_requests ?? 0}
+ hint="Background crawling: pages collected to train a model or to build an answer-engine index. Nobody is waiting on the other end."
+ isLoading={isLoading}
+ />
+ <AiStat
+ label="Visits sent back"
+ value={referrals}
+ hint="People who arrived on your site from an AI product in this period. Compare against crawls to see which operators give something back."
+ isLoading={isSummaryLoading}
+ />
+ </div>
+ </CardContent>
+ </Card>
+ );
+}
diff --git a/client/src/app/[site]/bots/components/ai/AiPages.tsx b/client/src/app/[site]/bots/components/ai/AiPages.tsx
new file mode 100644
index 000000000..bc1063190
--- /dev/null
+++ b/client/src/app/[site]/bots/components/ai/AiPages.tsx
@@ -0,0 +1,48 @@
+"use client";
+
+import { useGetSite } from "../../../../../api/admin/hooks/useSites";
+import { truncateString } from "../../../../../lib/utils";
+import { BotSectionTabs, type BotSectionTab } from "../BotSectionTabs";
+
+type Tab = "pages" | "agent_pages";
+
+/** What the AI systems actually read. */
+export function AiPages() {
+ const { data: siteMetadata } = useGetSite();
+
+ const linkFor = (host: string | undefined, path: string) =>
+ host && path ? `https://${host}${path}` : undefined;
+
+ const tabs: BotSectionTab<Tab>[] = [
+ {
+ value: "pages",
+ label: "All AI",
+ section: {
+ dimension: "pathname",
+ title: "Pages read by AI",
+ purpose: "ai",
+ getValue: item => item.value,
+ getKey: item => item.value || "unknown",
+ getLabel: item => truncateString(item.value, 50) || "Other",
+ getLink: item => linkFor(item.hostname || siteMetadata?.domain, item.value),
+ },
+ },
+ {
+ value: "agent_pages",
+ label: "Agents only",
+ section: {
+ dimension: "pathname",
+ title: "Pages agents opened",
+ // The pages a person's assistant was sent to are a different signal
+ // from the pages a crawler swept: these track live intent.
+ purpose: "ai_agent",
+ getValue: item => item.value,
+ getKey: item => item.value || "unknown",
+ getLabel: item => truncateString(item.value, 50) || "Other",
+ getLink: item => linkFor(item.hostname || siteMetadata?.domain, item.value),
+ },
+ },
+ ];
+
+ return <BotSectionTabs defaultValue="pages" tabs={tabs} />;
+}
diff --git a/client/src/app/[site]/bots/components/ai/aiLabels.ts b/client/src/app/[site]/bots/components/ai/aiLabels.ts
new file mode 100644
index 000000000..39137346b
--- /dev/null
+++ b/client/src/app/[site]/bots/components/ai/aiLabels.ts
@@ -0,0 +1,38 @@
+import type { BotPurpose } from "../../../../../api/analytics/endpoints";
+
+/** Colors for the two halves of AI traffic, used by the chart and the legend. */
+export const AI_AGENT_COLOR = "hsl(var(--dataviz))";
+export const AI_CRAWLER_COLOR = "hsl(var(--amber-400))";
+
+const PURPOSE_LABELS: Record<string, string> = {
+ ai_training: "AI training crawler",
+ ai_search: "AI answer engine",
+ ai_agent: "AI agent",
+ search: "Search engine",
+ social_preview: "Link preview",
+ seo: "SEO crawler",
+ monitoring: "Monitoring",
+ security: "Security scanner",
+ scripted: "Scripted client",
+ headless: "Headless browser",
+};
+
+const PURPOSE_DESCRIPTIONS: Record<string, string> = {
+ ai_training: "Collecting pages to train a model. Does not send readers back.",
+ ai_search: "Indexing pages so an assistant can cite them. Can send readers back.",
+ ai_agent: "Someone asked an assistant to open this page, just now.",
+};
+
+/**
+ * Rows written before bot identity shipped carry an empty purpose. Saying so is
+ * more honest than folding them into a real category.
+ */
+export function formatBotPurpose(value: string) {
+ return PURPOSE_LABELS[value] ?? (value ? value : "Unclassified");
+}
+
+export function describeBotPurpose(value: string) {
+ return PURPOSE_DESCRIPTIONS[value as BotPurpose];
+}
+
+export const AI_PURPOSE_ORDER: BotPurpose[] = ["ai_agent", "ai_search", "ai_training"];
diff --git a/client/src/app/[site]/bots/components/sections/BotMetadata.tsx b/client/src/app/[site]/bots/components/sections/BotMetadata.tsx
index fd5bf13e6..d5e578be0 100644
--- a/client/src/app/[site]/bots/components/sections/BotMetadata.tsx
+++ b/client/src/app/[site]/bots/components/sections/BotMetadata.tsx
@@ -2,8 +2,9 @@
import { truncateString } from "../../../../../lib/utils";
import { BotSectionTabs, type BotSectionTab } from "../BotSectionTabs";
+import { formatBotPurpose } from "../ai/aiLabels";
-type Tab = "asn_orgs" | "bot_categories" | "ua_patterns";
+type Tab = "bots" | "operators" | "purposes" | "asn_orgs" | "bot_categories" | "ua_patterns";
function formatBotCategory(value: string) {
if (!value) return "Uncategorized";
@@ -16,6 +17,45 @@ function formatBotCategory(value: string) {
export function BotMetadata() {
const tabs: BotSectionTab<Tab>[] = [
+ {
+ value: "bots",
+ label: "Bots",
+ section: {
+ dimension: "bot_name",
+ title: "Bots",
+ getValue: item => item.value,
+ getKey: item => item.value || "unnamed",
+ // Only the curated patterns carry a name. A hit on a generic rule is a
+ // bot nobody has identified, and rows written before identity shipped
+ // land here too.
+ getLabel: item => item.value || "Unnamed",
+ filterable: false,
+ },
+ },
+ {
+ value: "operators",
+ label: "Operators",
+ section: {
+ dimension: "bot_operator",
+ title: "Operators",
+ getValue: item => item.value,
+ getKey: item => item.value || "unknown",
+ getLabel: item => item.value || "Unknown",
+ filterable: false,
+ },
+ },
+ {
+ value: "purposes",
+ label: "Purpose",
+ section: {
+ dimension: "bot_purpose",
+ title: "Purpose",
+ getValue: item => item.value,
+ getKey: item => item.value || "unclassified",
+ getLabel: item => formatBotPurpose(item.value),
+ filterable: false,
+ },
+ },
{
value: "asn_orgs",
label: "ASN Orgs",
@@ -54,5 +94,5 @@ export function BotMetadata() {
},
];
- return <BotSectionTabs defaultValue="asn_orgs" tabs={tabs} />;
+ return <BotSectionTabs defaultValue="bots" tabs={tabs} />;
}
diff --git a/client/src/app/[site]/bots/page.tsx b/client/src/app/[site]/bots/page.tsx
index 433575c24..881e2fb21 100644
--- a/client/src/app/[site]/bots/page.tsx
+++ b/client/src/app/[site]/bots/page.tsx
@@ -2,17 +2,25 @@
import { ReactNode } from "react";
import { BOT_AVAILABLE_FILTERS } from "../../../api/analytics/hooks/bots/constants";
+import { Tabs, TabsList, TabsTrigger } from "../../../components/ui/tabs";
import { useInView } from "../../../hooks/useInView";
import { useSetPageTitle } from "../../../hooks/useSetPageTitle";
import { useStore } from "../../../lib/store";
import { SubHeader } from "../components/SubHeader/SubHeader";
import { BotChart } from "./components/BotChart";
import { BotsOverview } from "./components/BotsOverview";
+import { AiBots } from "./components/ai/AiBots";
+import { AiChart } from "./components/ai/AiChart";
+import { AiCoverageNote } from "./components/ai/AiCoverageNote";
+import { AiOperatorTable } from "./components/ai/AiOperatorTable";
+import { AiOverview } from "./components/ai/AiOverview";
+import { AiPages } from "./components/ai/AiPages";
import { BotCountries } from "./components/sections/BotCountries";
import { BotDevices } from "./components/sections/BotDevices";
import { BotMetadata } from "./components/sections/BotMetadata";
import { BotPages } from "./components/sections/BotPages";
import { BotReferrers } from "./components/sections/BotReferrers";
+import { useBotsStore } from "./botsStore";
function LazySection({ children, height = "405px" }: { children: ReactNode; height?: string }) {
const { ref, isInView } = useInView({ persistVisibility: true, rootMargin: "100px 0px" });
@@ -23,17 +31,35 @@ function LazySection({ children, height = "405px" }: { children: ReactNode; heig
);
}
-export default function BotsPage() {
- const { site } = useStore();
- useSetPageTitle("Bots");
-
- if (!site) {
- return null;
- }
+/**
+ * Which AI systems read the site, what they read, and what they sent back.
+ * Leads the page because it is the question people arrive with; the detection
+ * layers below answer a different one β€” how the traffic was caught.
+ */
+function AiLens() {
+ return (
+ <>
+ <div className="space-y-4">
+ <AiOverview />
+ <AiCoverageNote />
+ <AiChart />
+ <AiOperatorTable />
+ </div>
+ <div className="grid grid-cols-1 lg:grid-cols-2 gap-3 mt-3">
+ <LazySection>
+ <AiBots />
+ </LazySection>
+ <LazySection>
+ <AiPages />
+ </LazySection>
+ </div>
+ </>
+ );
+}
+function AllBotsLens() {
return (
- <div className="p-2 md:p-4 max-w-[1100px] mx-auto space-y-3">
- <SubHeader availableFilters={BOT_AVAILABLE_FILTERS} />
+ <>
<div className="space-y-4">
<BotsOverview />
<BotChart />
@@ -55,6 +81,29 @@ export default function BotsPage() {
<BotMetadata />
</LazySection>
</div>
+ </>
+ );
+}
+
+export default function BotsPage() {
+ const { site } = useStore();
+ const { lens, setLens } = useBotsStore();
+ useSetPageTitle("Bots");
+
+ if (!site) {
+ return null;
+ }
+
+ return (
+ <div className="p-2 md:p-4 max-w-[1100px] mx-auto space-y-3">
+ <SubHeader availableFilters={BOT_AVAILABLE_FILTERS} />
+ <Tabs value={lens} onValueChange={value => setLens(value as typeof lens)}>
+ <TabsList>
+ <TabsTrigger value="ai">AI &amp; agents</TabsTrigger>
+ <TabsTrigger value="all">All bots</TabsTrigger>
+ </TabsList>
+ </Tabs>
+ {lens === "ai" ? <AiLens /> : <AllBotsLens />}
</div>
);
}
diff --git a/docs/content/docs/(docs)/bot-detection.mdx b/docs/content/docs/(docs)/bot-detection.mdx
index 25cd82a96..80278dc64 100644
--- a/docs/content/docs/(docs)/bot-detection.mdx
+++ b/docs/content/docs/(docs)/bot-detection.mdx
@@ -125,6 +125,41 @@ This layer is designed to catch fast crawlers, floods, and replayed tracking req
</Steps>
+## Bot Identity
+
+Every detected bot carries the layers that caught it. Bots matched by one of Rybbit's curated user-agent patterns carry three more fields, which is what the Bots dashboard groups by:
+
+| Field | Example | Meaning |
+| --- | --- | --- |
+| `bot_name` | `GPTBot` | The name the bot publishes in its user agent |
+| `bot_operator` | `OpenAI` | Who runs it |
+| `bot_purpose` | `ai_training` | What the fetch is for |
+
+A bot matched only by a broad pattern (`crawl`, `spider`) is still filtered, but has no published identity β€” those rows read as **Unnamed**, which is more useful than a guess.
+
+### Purposes
+
+`bot_purpose` answers a different question from `bot_category`. Category groups bots into families; purpose says what the operator does with the page, which is the distinction that matters for AI traffic:
+
+| Purpose | What it means |
+| --- | --- |
+| `ai_training` | Collecting pages to train a model. Sends no readers back. Examples: GPTBot, ClaudeBot, CCBot, Google-Extended, Bytespider |
+| `ai_search` | Indexing pages so an assistant can cite them. Can send readers back. Examples: OAI-SearchBot, Claude-SearchBot, PerplexityBot |
+| `ai_agent` | A person asked an assistant to open the page, and it fetched it there and then. Examples: ChatGPT-User, Claude-User, Perplexity-User, Google-Agent, Claude Code |
+| `search` | Classic search engine indexing |
+| `social_preview` | Link unfurling |
+| `seo` | Backlink, rank, and site-audit crawlers |
+| `monitoring` | Uptime, synthetic, and performance checks |
+| `security` | Scanners |
+| `scripted` | HTTP libraries and CLI clients |
+| `headless` | Browser automation |
+
+Splitting the AI family in three is the point: a training crawler reading 4,000 pages and a person's assistant opening one page are both `category: ai`, but they mean opposite things for your site.
+
+<Callout>
+These fields were added in a later release. Bot events recorded before then have empty identity fields and appear as **Unnamed** / **Unclassified**; nothing is backfilled.
+</Callout>
+
## How Decisions Are Made
Rybbit does not stop at the first matching layer. It runs every layer, records all matches, and then makes one final decision.
@@ -141,7 +176,26 @@ Because multiple layers can match the same request, per-layer bot counts can add
## Server-Side Tracking
-Requests sent to `/api/track` with `Authorization: Bearer <api key>` and a valid API key are treated as trusted server-side ingestion and bypass bot blocking. Use this for backend-generated events where the request is coming from your server rather than a visitor's browser.
+Requests sent to `/api/track` with `Authorization: Bearer <api key>` and a valid API key are treated as trusted server-side ingestion. Four of the five layers are skipped: the headers, the client signals, the network address, and the request rate all belong to your server, not to the visitor it is reporting, so scoring them would convict your own backend.
+
+The user-agent layer still runs, on the `user_agent` you send. Nothing changes for ordinary backend events β€” a request reporting a real browser's user agent is tracked normally.
+
+### Reporting crawlers that never run JavaScript
+
+Rybbit's tracker is JavaScript, so a crawler that fetches your HTML and executes nothing never reaches Rybbit at all. Most AI training crawlers work exactly that way, which is why the Bots dashboard carries a note about its own coverage.
+
+If you want to see them, report them from the edge β€” middleware, a worker, or a reverse proxy β€” forwarding the real request's `user_agent`:
+
+```json
+{
+ "site_id": "1",
+ "hostname": "example.com",
+ "pathname": "/pricing",
+ "user_agent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.2; +https://openai.com/gptbot"
+}
+```
+
+A forwarded crawler user agent is classified and lands in your bot events with full identity, rather than in your analytics as a phantom visitor.
<Callout type="warn">
Do not expose API keys in browser JavaScript. API keys are only for server-side requests.
diff --git a/docs/content/docs/api/bots/ai-summary.mdx b/docs/content/docs/api/bots/ai-summary.mdx
new file mode 100644
index 000000000..4094c865c
--- /dev/null
+++ b/docs/content/docs/api/bots/ai-summary.mdx
@@ -0,0 +1,162 @@
+---
+title: Get AI Summary
+description: Returns crawls and referrals per AI operator, side by side.
+full: true
+method: GET
+---
+
+import { TypeTable } from 'fumadocs-ui/components/type-table';
+import { Tab, Tabs } from 'fumadocs-ui/components/tabs';
+import { Callout } from 'fumadocs-ui/components/callout';
+
+<div className="grid grid-cols-1 xl:grid-cols-2 gap-8">
+<div>
+
+```
+GET /api/sites/:site/bots/ai-summary
+```
+
+Returns one row per AI operator: how much of your site it read, and how many visits it sent back.
+
+The two numbers come from different places. Reads are bot events whose user agent names the operator; visits back are sessions in your normal analytics that arrived from that company's product (the `AI` acquisition channel). Neither is very interesting alone β€” "OpenAI read 4,120 pages and sent 12 visits" is the sentence this endpoint exists to produce.
+
+Operators that only read, and operators that only referred, both appear; the missing side reads `0`.
+
+### Path Parameters
+
+<TypeTable
+ type={{
+ site: {
+ description: 'Site ID or site identifier',
+ type: 'string | number',
+ required: true,
+ },
+ }}
+/>
+
+### Query Parameters
+
+Accepts all [Common Parameters](/docs/api/getting-started#common-parameters) (time and filters).
+
+Filters apply to both halves, using the bot filter subset described in [Get Bots by Dimension](/docs/api/bots/by-dimension).
+
+### Response
+
+<TypeTable
+ type={{
+ data: {
+ description: 'Array of operator rows, ordered by reads descending',
+ type: 'AiSummaryRow[]',
+ },
+ }}
+/>
+
+#### AiSummaryRow Object
+
+<TypeTable
+ type={{
+ operator: {
+ description: 'Who runs the bot, e.g. "OpenAI"',
+ type: 'string',
+ },
+ crawls: {
+ description: 'Total AI requests from this operator',
+ type: 'number',
+ },
+ training_crawls: {
+ description: 'Requests with purpose ai_training',
+ type: 'number',
+ },
+ search_crawls: {
+ description: 'Requests with purpose ai_search',
+ type: 'number',
+ },
+ agent_requests: {
+ description: 'Requests with purpose ai_agent',
+ type: 'number',
+ },
+ referrals: {
+ description: 'Sessions that arrived from this operator\'s product',
+ type: 'number',
+ },
+ crawls_per_referral: {
+ description: 'Reads per visit sent back. 0 when the operator sent nobody back β€” there is no rate to report.',
+ type: 'number',
+ },
+ }}
+/>
+
+<Callout>
+Only bots that execute JavaScript reach Rybbit. Crawlers that fetch HTML and run nothing β€” which includes most training crawlers β€” are absent unless your server reports them. See [Bot detection](/docs/bot-detection#reporting-crawlers-that-never-run-javascript).
+</Callout>
+
+</div>
+<div className="flex flex-col gap-4">
+
+<Tabs items={['cURL', 'JavaScript', 'Python']}>
+ <Tab value="cURL">
+```bash title="Request"
+curl -X GET "https://app.rybbit.io/api/sites/123/bots/ai-summary?start_date=2024-01-01&end_date=2024-01-31&time_zone=America/New_York" \
+ -H "Authorization: Bearer your_api_key_here"
+```
+ </Tab>
+ <Tab value="JavaScript">
+```javascript title="Request"
+const response = await fetch(
+ 'https://app.rybbit.io/api/sites/123/bots/ai-summary?start_date=2024-01-01&end_date=2024-01-31&time_zone=America/New_York',
+ {
+ headers: {
+ 'Authorization': 'Bearer your_api_key_here'
+ }
+ }
+);
+
+const data = await response.json();
+```
+ </Tab>
+ <Tab value="Python">
+```python title="Request"
+import requests
+
+response = requests.get(
+ 'https://app.rybbit.io/api/sites/123/bots/ai-summary',
+ params={
+ 'start_date': '2024-01-01',
+ 'end_date': '2024-01-31',
+ 'time_zone': 'America/New_York',
+ },
+ headers={'Authorization': 'Bearer your_api_key_here'},
+)
+
+data = response.json()
+```
+ </Tab>
+</Tabs>
+
+```json title="Response"
+{
+ "data": [
+ {
+ "operator": "OpenAI",
+ "crawls": 6120,
+ "training_crawls": 4400,
+ "search_crawls": 900,
+ "agent_requests": 820,
+ "referrals": 214,
+ "crawls_per_referral": 28.6
+ },
+ {
+ "operator": "Perplexity",
+ "crawls": 1904,
+ "training_crawls": 0,
+ "search_crawls": 1300,
+ "agent_requests": 604,
+ "referrals": 341,
+ "crawls_per_referral": 5.6
+ }
+ ]
+}
+```
+
+</div>
+</div>
diff --git a/docs/content/docs/api/bots/by-dimension.mdx b/docs/content/docs/api/bots/by-dimension.mdx
index 1c70f6f9f..24c823a80 100644
--- a/docs/content/docs/api/bots/by-dimension.mdx
+++ b/docs/content/docs/api/bots/by-dimension.mdx
@@ -40,7 +40,7 @@ Bot filtering supports a subset of filter parameters: `browser`, `browser_versio
dimension: {
description: 'The dimension to group by',
type: 'string',
- typeDescription: '"browser" | "browser_version" | "operating_system" | "operating_system_version" | "country" | "region" | "city" | "device_type" | "referrer" | "hostname" | "pathname" | "dimensions" | "asn_org" | "bot_category" | "matched_ua_pattern"',
+ typeDescription: '"browser" | "browser_version" | "operating_system" | "operating_system_version" | "country" | "region" | "city" | "device_type" | "referrer" | "hostname" | "pathname" | "dimensions" | "asn_org" | "asn_provider" | "bot_category" | "bot_name" | "bot_operator" | "bot_purpose" | "matched_ua_pattern"',
required: true,
},
limit: {
@@ -53,8 +53,14 @@ Bot filtering supports a subset of filter parameters: `browser`, `browser_versio
type: 'number',
default: 1,
},
+ purpose: {
+ description: 'Restrict results to one bot purpose. "ai" covers all three AI purposes; "ai_crawler" covers training and answer-engine crawling but not agents.',
+ type: 'string',
+ typeDescription: '"ai" | "ai_crawler" | "ai_training" | "ai_search" | "ai_agent" | "search" | "social_preview" | "seo" | "monitoring" | "security" | "scripted" | "headless"',
+ required: false,
+ },
layer: {
- description: 'Restrict results to a single detection layer',
+ description: 'Restrict results to a single detection layer
type: 'string',
typeDescription: '"ua_pattern" | "header_heuristics" | "client_signals" | "bot_asn" | "rate_anomaly"',
required: false,
diff --git a/docs/content/docs/api/bots/meta.json b/docs/content/docs/api/bots/meta.json
index 2dfff9cb9..717cb3873 100644
--- a/docs/content/docs/api/bots/meta.json
+++ b/docs/content/docs/api/bots/meta.json
@@ -1,4 +1,9 @@
{
"title": "Bots",
- "pages": ["overview", "time-series", "by-dimension"]
+ "pages": [
+ "overview",
+ "time-series",
+ "by-dimension",
+ "ai-summary"
+ ]
}
diff --git a/docs/content/docs/api/bots/overview.mdx b/docs/content/docs/api/bots/overview.mdx
index 58da59688..f8cbb6452 100644
--- a/docs/content/docs/api/bots/overview.mdx
+++ b/docs/content/docs/api/bots/overview.mdx
@@ -63,6 +63,18 @@ Bot filtering supports a subset of filter parameters: `browser`, `browser_versio
type: 'number',
typeDescription: 'Range: 0-100',
},
+ ai_requests: {
+ description: 'Requests from named AI systems: training crawlers, answer-engine indexers, and user-triggered agents',
+ type: 'number',
+ },
+ ai_agent_requests: {
+ description: 'Requests from user-triggered AI agents (a person asked an assistant to open the page)',
+ type: 'number',
+ },
+ ai_crawler_requests: {
+ description: 'Requests from AI training crawlers and answer-engine indexers',
+ type: 'number',
+ },
ua_pattern: {
description: 'Bot requests flagged by the user-agent pattern layer',
type: 'number',
@@ -211,6 +223,9 @@ var data = await response.Content.ReadAsStringAsync();
"bot_requests": 1840,
"total_events": 48933,
"bot_percentage": 3.76,
+ "ai_requests": 940,
+ "ai_agent_requests": 180,
+ "ai_crawler_requests": 760,
"ua_pattern": 1200,
"header_heuristics": 410,
"client_signals": 95,
diff --git a/docs/content/docs/api/bots/time-series.mdx b/docs/content/docs/api/bots/time-series.mdx
index 8458568fb..1a8bab44f 100644
--- a/docs/content/docs/api/bots/time-series.mdx
+++ b/docs/content/docs/api/bots/time-series.mdx
@@ -43,6 +43,12 @@ Bot filtering supports a subset of filter parameters: `browser`, `browser_versio
typeDescription: '"minute" | "five_minutes" | "ten_minutes" | "fifteen_minutes" | "hour" | "day" | "week" | "month" | "year"',
default: 'hour',
},
+ purpose: {
+ description: 'Restrict results to one bot purpose. "ai" covers all three AI purposes; "ai_crawler" covers training and answer-engine crawling but not agents.',
+ type: 'string',
+ typeDescription: '"ai" | "ai_crawler" | "ai_training" | "ai_search" | "ai_agent" | "search" | "social_preview" | "seo" | "monitoring" | "security" | "scripted" | "headless"',
+ required: false,
+ },
layer: {
description: 'Restrict results to a single detection layer',
type: 'string',
@@ -75,6 +81,14 @@ Bot filtering supports a subset of filter parameters: `browser`, `browser_versio
description: 'Number of bot requests in this bucket',
type: 'number',
},
+ ai_agent_requests: {
+ description: 'Requests in this bucket from user-triggered AI agents',
+ type: 'number',
+ },
+ ai_crawler_requests: {
+ description: 'Requests in this bucket from AI training crawlers and answer-engine indexers',
+ type: 'number',
+ },
}}
/>
diff --git a/server/src/api/analytics/bots/getBotAiSummary.test.ts b/server/src/api/analytics/bots/getBotAiSummary.test.ts
new file mode 100644
index 000000000..60e964f61
--- /dev/null
+++ b/server/src/api/analytics/bots/getBotAiSummary.test.ts
@@ -0,0 +1,56 @@
+import { describe, expect, it } from "vitest";
+import { buildBotAiSummaryQuery } from "./getBotAiSummary.js";
+
+const baseQuery = { start_date: "2024-01-01", end_date: "2024-01-31", time_zone: "UTC" } as any;
+
+describe("buildBotAiSummaryQuery", () => {
+ it("counts crawls per operator, split by purpose", () => {
+ const sql = buildBotAiSummaryQuery(baseQuery);
+
+ expect(sql).toContain("bot_operator AS operator");
+ expect(sql).toContain("countIf(bot_purpose = 'ai_training') AS training_crawls");
+ expect(sql).toContain("countIf(bot_purpose = 'ai_search') AS search_crawls");
+ expect(sql).toContain("countIf(bot_purpose = 'ai_agent') AS agent_requests");
+ });
+
+ it("excludes non-AI bots and rows written before identity shipped", () => {
+ const sql = buildBotAiSummaryQuery(baseQuery);
+
+ // An SEO or monitoring bot has an operator too; only the AI purposes belong
+ // on this surface. Pre-identity rows carry '' and would otherwise group
+ // into one meaningless bucket.
+ expect(sql).toContain("AND bot_operator != ''");
+ expect(sql).toContain("AND bot_purpose IN ('ai_training', 'ai_search', 'ai_agent')");
+ });
+
+ it("maps AI referrer domains onto the same operator names the bot patterns use", () => {
+ const sql = buildBotAiSummaryQuery(baseQuery);
+
+ expect(sql).toContain("'chatgpt.com'");
+ expect(sql).toContain("'OpenAI'");
+ expect(sql).toContain("domainWithoutWWW(referrer)");
+ expect(sql).toContain("AND channel = 'AI'");
+ });
+
+ it("does not treat duckduckgo as an AI referrer", () => {
+ // DuckAssistBot crawls, but a duckduckgo.com referral is organic search.
+ // Listing it would silently reclassify every DuckDuckGo visit.
+ expect(buildBotAiSummaryQuery(baseQuery)).not.toContain("duckduckgo.com");
+ });
+
+ it("keeps operators that only crawl and operators that only refer", () => {
+ const sql = buildBotAiSummaryQuery(baseQuery);
+
+ expect(sql).toContain("FULL OUTER JOIN referrals ON crawls.operator = referrals.operator");
+ expect(sql).toContain("if(crawls.operator != '', crawls.operator, referrals.operator) AS operator");
+ });
+
+ it("guards the ratio against a zero denominator", () => {
+ expect(buildBotAiSummaryQuery(baseQuery)).toContain("if(referrals.referrals = 0, 0,");
+ });
+
+ it("applies the time window to both halves", () => {
+ const sql = buildBotAiSummaryQuery(baseQuery);
+ expect(sql.match(/toStartOfDay/g)?.length).toBeGreaterThanOrEqual(2);
+ });
+});
diff --git a/server/src/api/analytics/bots/getBotAiSummary.ts b/server/src/api/analytics/bots/getBotAiSummary.ts
new file mode 100644
index 000000000..7e1227970
--- /dev/null
+++ b/server/src/api/analytics/bots/getBotAiSummary.ts
@@ -0,0 +1,110 @@
+import { AI_REFERRER_DOMAIN_TO_OPERATOR, FilterParams } from "@rybbit/shared";
+import { FastifyReply, FastifyRequest } from "fastify";
+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";
+import { getFilterStatement } from "../utils/getFilterStatement.js";
+import { getTimeStatement } from "../utils/timeWindow.js";
+import { AI_PURPOSE_SQL_LIST, getBotFilterStatement } from "./utils.js";
+
+/**
+ * One row per AI operator: how much of the site it read, and how much traffic
+ * it sent back.
+ *
+ * Crawl counts and referral counts come from two different tables written by
+ * two unrelated code paths β€” `bot_events` during bot detection, `events`
+ * during channel classification β€” and are only meaningful next to each other.
+ * "OpenAI fetched 4,120 pages and sent 12 visits" is a sentence a site owner
+ * can act on; either number alone is trivia.
+ */
+type BotAiSummaryRow = {
+ operator: string;
+ crawls: number;
+ training_crawls: number;
+ search_crawls: number;
+ agent_requests: number;
+ referrals: number;
+ /** Crawls per referral. 0 when the operator sent no one back. */
+ crawls_per_referral: number;
+};
+
+export interface BotAiSummaryRequest {
+ Params: {
+ siteId: string;
+ };
+ Querystring: FilterParams<{}>;
+}
+
+/**
+ * ClickHouse `transform()` mapping a referrer domain onto the operator that
+ * runs it, spelled exactly as the bot patterns spell it so the two sides join.
+ */
+function buildReferrerOperatorExpression() {
+ const entries = Object.entries(AI_REFERRER_DOMAIN_TO_OPERATOR);
+ const domains = entries.map(([domain]) => `'${domain}'`).join(", ");
+ const operators = entries.map(([, operator]) => `'${operator}'`).join(", ");
+ return `transform(domainWithoutWWW(referrer), [${domains}], [${operators}], '')`;
+}
+
+export const buildBotAiSummaryQuery = (query: BotAiSummaryRequest["Querystring"]) => {
+ const timeStatement = getTimeStatement(query);
+ const botFilterStatement = getBotFilterStatement(query.filters);
+ // The referral half reads `events`, so it takes the events-surface filter
+ // builder rather than the bot-table one.
+ const eventFilterStatement = getFilterStatement(query.filters || "");
+ const referrerOperator = buildReferrerOperatorExpression();
+
+ return `
+ WITH
+ crawls AS (
+ SELECT
+ bot_operator AS operator,
+ count() AS crawls,
+ countIf(bot_purpose = 'ai_training') AS training_crawls,
+ countIf(bot_purpose = 'ai_search') AS search_crawls,
+ countIf(bot_purpose = 'ai_agent') AS agent_requests
+ FROM bot_events
+ WHERE site_id = {siteId:Int32}
+ AND bot_operator != ''
+ AND bot_purpose IN (${AI_PURPOSE_SQL_LIST})
+ ${botFilterStatement}
+ ${timeStatement}
+ GROUP BY operator
+ ),
+ referrals AS (
+ SELECT
+ ${referrerOperator} AS operator,
+ count() AS referrals
+ FROM events
+ WHERE site_id = {siteId:Int32}
+ AND channel = 'AI'
+ ${eventFilterStatement}
+ ${timeStatement}
+ GROUP BY operator
+ HAVING operator != ''
+ )
+ SELECT
+ -- FULL OUTER JOIN so an operator that only crawls and one that only
+ -- refers both survive; the empty side of the join reads 0, not absent.
+ if(crawls.operator != '', crawls.operator, referrals.operator) AS operator,
+ crawls.crawls AS crawls,
+ crawls.training_crawls AS training_crawls,
+ crawls.search_crawls AS search_crawls,
+ crawls.agent_requests AS agent_requests,
+ referrals.referrals AS referrals,
+ if(referrals.referrals = 0, 0, round(crawls.crawls / referrals.referrals, 1)) AS crawls_per_referral
+ FROM crawls
+ FULL OUTER JOIN referrals ON crawls.operator = referrals.operator
+ ORDER BY crawls DESC, referrals DESC
+ `;
+};
+
+export const getBotAiSummary = analyticsRoute<BotAiSummaryRequest>(
+ "bot ai summary",
+ async (req: FastifyRequest<BotAiSummaryRequest>, res: FastifyReply) => {
+ const data = await runAnalyticsQuery<BotAiSummaryRow>({
+ query: buildBotAiSummaryQuery(req.query),
+ params: { siteId: Number(req.params.siteId) },
+ });
+
+ return res.send({ data });
+ }
+);
diff --git a/server/src/api/analytics/bots/getBotDimension.ts b/server/src/api/analytics/bots/getBotDimension.ts
index c91ac1d1f..35ad264be 100644
--- a/server/src/api/analytics/bots/getBotDimension.ts
+++ b/server/src/api/analytics/bots/getBotDimension.ts
@@ -8,6 +8,7 @@ import {
type BotLayerKey,
getBotFilterStatement,
getBotLayerStatement,
+ getBotPurposeStatement,
getBotSqlParam,
} from "./utils.js";
@@ -30,6 +31,8 @@ export interface BotDimensionRequest {
Querystring: FilterParams<{
dimension: BotDimensionKey;
layer?: BotLayerKey;
+ /** A single purpose, or "ai" / "ai_crawler" for the grouped families. */
+ purpose?: string;
limit?: number;
page?: number;
}>;
@@ -45,6 +48,7 @@ export const buildBotDimensionQuery = (query: BotDimensionRequest["Querystring"]
const timeStatement = getTimeStatement(query);
const filterStatement = getBotFilterStatement(query.filters);
const layerStatement = getBotLayerStatement(query.layer);
+ const purposeStatement = getBotPurposeStatement(query.purpose);
const { limitStatement, offsetStatement } = getPaginationStatements(query, 100, isCountQuery);
const groupedQuery = `
@@ -57,6 +61,7 @@ export const buildBotDimensionQuery = (query: BotDimensionRequest["Querystring"]
WHERE site_id = {siteId:Int32}
${filterStatement}
${layerStatement}
+ ${purposeStatement}
${timeStatement}
GROUP BY value
`;
diff --git a/server/src/api/analytics/bots/getBotOverview.ts b/server/src/api/analytics/bots/getBotOverview.ts
index 3a92d68ef..f2f6b94bd 100644
--- a/server/src/api/analytics/bots/getBotOverview.ts
+++ b/server/src/api/analytics/bots/getBotOverview.ts
@@ -2,12 +2,21 @@ import { FilterParams } from "@rybbit/shared";
import { FastifyReply, FastifyRequest } from "fastify";
import { getTimeStatement } from "../utils/timeWindow.js";
import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";
-import { type BotLayerKey, getBotFilterStatement, getBotLayerStatement } from "./utils.js";
+import {
+ AI_CRAWLER_PURPOSE_SQL_LIST,
+ AI_PURPOSE_SQL_LIST,
+ type BotLayerKey,
+ getBotFilterStatement,
+ getBotLayerStatement,
+} from "./utils.js";
type BotOverviewResponse = {
bot_requests: number;
total_events: number;
bot_percentage: number;
+ ai_requests: number;
+ ai_agent_requests: number;
+ ai_crawler_requests: number;
ua_pattern: number;
header_heuristics: number;
client_signals: number;
@@ -38,7 +47,12 @@ export const buildBotOverviewQuery = (query: BotOverviewRequest["Querystring"])
countIf(detected_header_heuristics) AS header_heuristics,
countIf(detected_client_signals) AS client_signals,
countIf(detected_bot_asn) AS bot_asn,
- countIf(detected_rate_anomaly) AS rate_anomaly
+ countIf(detected_rate_anomaly) AS rate_anomaly,
+ -- Purpose is only set on rows written since bot identity shipped, so
+ -- these read 0 for older windows rather than being wrong.
+ countIf(bot_purpose IN (${AI_PURPOSE_SQL_LIST})) AS ai_requests,
+ countIf(bot_purpose = 'ai_agent') AS ai_agent_requests,
+ countIf(bot_purpose IN (${AI_CRAWLER_PURPOSE_SQL_LIST})) AS ai_crawler_requests
FROM bot_events
WHERE site_id = {siteId:Int32}
${filterStatement}
@@ -71,7 +85,10 @@ export const buildBotOverviewQuery = (query: BotOverviewRequest["Querystring"])
header_heuristics,
client_signals,
bot_asn,
- rate_anomaly
+ rate_anomaly,
+ ai_requests,
+ ai_agent_requests,
+ ai_crawler_requests
FROM bot_stats
CROSS JOIN all_bot_stats
CROSS JOIN event_stats
diff --git a/server/src/api/analytics/bots/getBotTimeSeries.ts b/server/src/api/analytics/bots/getBotTimeSeries.ts
index d6f035f4d..18d123b94 100644
--- a/server/src/api/analytics/bots/getBotTimeSeries.ts
+++ b/server/src/api/analytics/bots/getBotTimeSeries.ts
@@ -3,11 +3,20 @@ import { FastifyReply, FastifyRequest } from "fastify";
import { TimeBucket } from "../types.js";
import { resolveTimeWindow } from "../utils/timeWindow.js";
import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";
-import { type BotLayerKey, getBotFilterStatement, getBotLayerStatement } from "./utils.js";
+import {
+ AI_CRAWLER_PURPOSE_SQL_LIST,
+ AI_PURPOSE_SQL_LIST,
+ type BotLayerKey,
+ getBotFilterStatement,
+ getBotLayerStatement,
+ getBotPurposeStatement,
+} from "./utils.js";
type BotTimeSeriesPoint = {
time: string;
bot_requests: number;
+ ai_agent_requests: number;
+ ai_crawler_requests: number;
};
export interface BotTimeSeriesRequest {
@@ -17,6 +26,8 @@ export interface BotTimeSeriesRequest {
Querystring: FilterParams<{
bucket: TimeBucket;
layer?: BotLayerKey;
+ /** A single purpose, or "ai" / "ai_crawler" for the grouped families. */
+ purpose?: string;
}>;
}
@@ -26,16 +37,22 @@ export const buildBotTimeSeriesQuery = (query: BotTimeSeriesRequest["Querystring
const timeStatement = window.where();
const filterStatement = getBotFilterStatement(query.filters);
const layerStatement = getBotLayerStatement(query.layer);
+ const purposeStatement = getBotPurposeStatement(query.purpose);
const fillClause = window.fill(bucket);
return `
SELECT
${window.bucketed("timestamp", bucket)} AS time,
- count() AS bot_requests
+ count() AS bot_requests,
+ -- Returned on every bucket so the chart can draw agents against crawlers
+ -- without a second round trip; both read 0 on windows predating identity.
+ countIf(bot_purpose = 'ai_agent') AS ai_agent_requests,
+ countIf(bot_purpose IN (${AI_CRAWLER_PURPOSE_SQL_LIST})) AS ai_crawler_requests
FROM bot_events
WHERE site_id = {siteId:Int32}
${filterStatement}
${layerStatement}
+ ${purposeStatement}
${timeStatement}
GROUP BY time
ORDER BY time ${fillClause}
diff --git a/server/src/api/analytics/bots/utils.test.ts b/server/src/api/analytics/bots/utils.test.ts
index 17340a452..a06a2cc1b 100644
--- a/server/src/api/analytics/bots/utils.test.ts
+++ b/server/src/api/analytics/bots/utils.test.ts
@@ -5,7 +5,13 @@ vi.mock("../../../db/postgres/postgres.js", () => ({
sql: {},
}));
-import { buildStringFilterCondition, getBotFilterStatement, getBotLayerStatement, getBotSqlParam } from "./utils.js";
+import {
+ buildStringFilterCondition,
+ getBotFilterStatement,
+ getBotLayerStatement,
+ getBotPurposeStatement,
+ getBotSqlParam,
+} from "./utils.js";
describe("getBotLayerStatement", () => {
it("should return empty string for undefined layer", () => {
@@ -283,3 +289,33 @@ describe("getBotFilterStatement", () => {
});
});
+
+describe("getBotPurposeStatement", () => {
+ it("returns nothing when no purpose is selected", () => {
+ expect(getBotPurposeStatement()).toBe("");
+ expect(getBotPurposeStatement(null)).toBe("");
+ expect(getBotPurposeStatement("")).toBe("");
+ });
+
+ it("expands the ai family to all three AI purposes", () => {
+ expect(getBotPurposeStatement("ai")).toBe("AND bot_purpose IN ('ai_training', 'ai_search', 'ai_agent')");
+ });
+
+ it("expands ai_crawler to the two non-agent AI purposes", () => {
+ // Agents are a person's fetch, not background crawling β€” the whole point of
+ // the split is that they do not belong in a crawler total.
+ expect(getBotPurposeStatement("ai_crawler")).toBe("AND bot_purpose IN ('ai_training', 'ai_search')");
+ });
+
+ it("matches a single known purpose", () => {
+ expect(getBotPurposeStatement("ai_agent")).toBe("AND bot_purpose = 'ai_agent'");
+ expect(getBotPurposeStatement("seo")).toBe("AND bot_purpose = 'seo'");
+ });
+
+ it("ignores an unknown purpose rather than interpolating it", () => {
+ // The value reaches SQL by interpolation, so anything not on the allowlist
+ // must produce no clause at all.
+ expect(getBotPurposeStatement("'; DROP TABLE bot_events; --")).toBe("");
+ expect(getBotPurposeStatement("nonsense")).toBe("");
+ });
+});
diff --git a/server/src/api/analytics/bots/utils.ts b/server/src/api/analytics/bots/utils.ts
index 727979f0c..a331f0420 100644
--- a/server/src/api/analytics/bots/utils.ts
+++ b/server/src/api/analytics/bots/utils.ts
@@ -14,7 +14,15 @@ export const BOT_LAYER_COLUMNS = {
} as const;
export type BotLayerKey = keyof typeof BOT_LAYER_COLUMNS;
-export type BotDimensionKey = FilterParameter | "asn_org" | "bot_category" | "matched_ua_pattern";
+export type BotDimensionKey =
+ | FilterParameter
+ | "asn_org"
+ | "asn_provider"
+ | "bot_category"
+ | "bot_name"
+ | "bot_operator"
+ | "bot_purpose"
+ | "matched_ua_pattern";
const BOT_FILTER_PARAMETERS = new Set<FilterParameter>([
"browser",
@@ -49,10 +57,58 @@ export const BOT_DIMENSIONS = new Set<BotDimensionKey>([
"pathname",
"dimensions",
"asn_org",
+ "asn_provider",
"bot_category",
+ "bot_name",
+ "bot_operator",
+ "bot_purpose",
"matched_ua_pattern",
]);
+/**
+ * Purposes that count as AI traffic. Grouped rather than enumerated at every
+ * call site so "AI" means one thing across the overview, the chart and every
+ * breakdown on the page.
+ */
+export const AI_BOT_PURPOSES = ["ai_training", "ai_search", "ai_agent"] as const;
+export const AI_CRAWLER_PURPOSES = ["ai_training", "ai_search"] as const;
+
+const quoteList = (values: readonly string[]) => values.map(value => `'${value}'`).join(", ");
+
+export const AI_PURPOSE_SQL_LIST = quoteList(AI_BOT_PURPOSES);
+export const AI_CRAWLER_PURPOSE_SQL_LIST = quoteList(AI_CRAWLER_PURPOSES);
+
+const BOT_PURPOSES = new Set<string>([
+ ...AI_BOT_PURPOSES,
+ "search",
+ "social_preview",
+ "seo",
+ "monitoring",
+ "security",
+ "scripted",
+ "headless",
+]);
+
+/**
+ * Narrows a bot query to one purpose, or to the whole AI family with `"ai"`.
+ *
+ * Purpose is not a filter parameter β€” it exists only on the bot tables and has
+ * no equivalent on the events surface β€” so it takes the same dedicated-clause
+ * route `layer` does rather than going through the filter allowlist.
+ */
+export function getBotPurposeStatement(purpose?: string | null) {
+ if (!purpose) {
+ return "";
+ }
+ if (purpose === "ai") {
+ return `AND bot_purpose IN (${AI_PURPOSE_SQL_LIST})`;
+ }
+ if (purpose === "ai_crawler") {
+ return `AND bot_purpose IN (${AI_CRAWLER_PURPOSE_SQL_LIST})`;
+ }
+ return BOT_PURPOSES.has(purpose) ? `AND bot_purpose = '${purpose}'` : "";
+}
+
export function getBotLayerStatement(layer?: string | null) {
if (!layer) {
return "";
@@ -64,7 +120,15 @@ export function getBotLayerStatement(layer?: string | null) {
// Dimension keys that only exist on bot_events; everything else shares the
// events-surface column expressions from getSqlParam.
-const BOT_ONLY_DIMENSIONS = new Set<BotDimensionKey>(["asn_org", "bot_category", "matched_ua_pattern"]);
+const BOT_ONLY_DIMENSIONS = new Set<BotDimensionKey>([
+ "asn_org",
+ "asn_provider",
+ "bot_category",
+ "bot_name",
+ "bot_operator",
+ "bot_purpose",
+ "matched_ua_pattern",
+]);
export const getBotSqlParam = (parameter: BotDimensionKey) => {
if (BOT_ONLY_DIMENSIONS.has(parameter)) {
diff --git a/server/src/api/analytics/index.ts b/server/src/api/analytics/index.ts
index c4ea5f069..43718b8de 100644
--- a/server/src/api/analytics/index.ts
+++ b/server/src/api/analytics/index.ts
@@ -37,6 +37,7 @@ export { getPerformanceOverview } from "./performance/getPerformanceOverview.js"
export { getPerformanceTimeSeries } from "./performance/getPerformanceTimeSeries.js";
// Bots
+export { getBotAiSummary } from "./bots/getBotAiSummary.js";
export { getBotDimension } from "./bots/getBotDimension.js";
export { getBotOverview } from "./bots/getBotOverview.js";
export { getBotTimeSeries } from "./bots/getBotTimeSeries.js";
diff --git a/server/src/db/clickhouse/schema/core.ts b/server/src/db/clickhouse/schema/core.ts
index 344ef1bce..c355608db 100644
--- a/server/src/db/clickhouse/schema/core.ts
+++ b/server/src/db/clickhouse/schema/core.ts
@@ -34,6 +34,17 @@ const BOT_EVENTS_COLUMNS_TO_ENSURE: ColumnDefinition[] = [
// say why the request was convicted.
{ name: "anomaly_reasons", definition: "anomaly_reasons String DEFAULT ''" },
{ name: "anomaly_score", definition: "anomaly_score UInt8 DEFAULT 0" },
+ // Bot identity. `matched_ua_pattern` stores a regex source, which is evidence
+ // but not an answer to "who is this" β€” these carry the name the operator
+ // publishes. Empty on rows written before the columns existed; nothing is
+ // backfilled, so readers must tolerate ''.
+ { name: "bot_name", definition: "bot_name LowCardinality(String) DEFAULT ''" },
+ { name: "bot_operator", definition: "bot_operator LowCardinality(String) DEFAULT ''" },
+ { name: "bot_purpose", definition: "bot_purpose LowCardinality(String) DEFAULT ''" },
+ // The curated provider behind the request's ASN, resolved during detection
+ // and until now discarded. A UA can claim any name; this is the independent
+ // half of the claim, so the two together are what makes attribution arguable.
+ { name: "asn_provider", definition: "asn_provider LowCardinality(String) DEFAULT ''" },
];
// Runs against both audit tables: they carry the same columns on purpose, so a
@@ -175,7 +186,11 @@ export async function initializeCoreTables() {
client_bot_score Nullable(UInt8),
client_signal_mask UInt16 DEFAULT 0,
anomaly_reasons String DEFAULT '',
- anomaly_score UInt8 DEFAULT 0
+ anomaly_score UInt8 DEFAULT 0,
+ bot_name LowCardinality(String) DEFAULT '',
+ bot_operator LowCardinality(String) DEFAULT '',
+ bot_purpose LowCardinality(String) DEFAULT '',
+ asn_provider LowCardinality(String) DEFAULT ''
)
ENGINE = MergeTree()
PARTITION BY toYYYYMM(timestamp)
@@ -227,7 +242,11 @@ export async function initializeCoreTables() {
client_bot_score Nullable(UInt8),
client_signal_mask UInt16 DEFAULT 0,
anomaly_reasons String DEFAULT '',
- anomaly_score UInt8 DEFAULT 0
+ anomaly_score UInt8 DEFAULT 0,
+ bot_name LowCardinality(String) DEFAULT '',
+ bot_operator LowCardinality(String) DEFAULT '',
+ bot_purpose LowCardinality(String) DEFAULT '',
+ asn_provider LowCardinality(String) DEFAULT ''
)
ENGINE = MergeTree()
PARTITION BY toYYYYMM(timestamp)
diff --git a/server/src/index.ts b/server/src/index.ts
index d93403ef6..c049fc137 100644
--- a/server/src/index.ts
+++ b/server/src/index.ts
@@ -31,6 +31,7 @@ import {
generatePdfReport,
getDashboard,
getDashboards,
+ getBotAiSummary,
getBotDimension,
getBotOverview,
getBotTimeSeries,
@@ -430,6 +431,7 @@ async function analyticsRoutes(fastify: FastifyInstance) {
fastify.get("/sites/:siteId/bots/overview", publicAnalyticsRead, getBotOverview);
fastify.get("/sites/:siteId/bots/time-series", publicAnalyticsRead, getBotTimeSeries);
fastify.get("/sites/:siteId/bots/by-dimension", publicAnalyticsRead, getBotDimension);
+ fastify.get("/sites/:siteId/bots/ai-summary", publicAnalyticsRead, getBotAiSummary);
fastify.get("/sites/:siteId/export/pdf", authAnalyticsRead, generatePdfReport);
}
diff --git a/server/src/services/tracker/botBlocking/botEventQueue.ts b/server/src/services/tracker/botBlocking/botEventQueue.ts
index 453b385f6..aa27eb9a8 100644
--- a/server/src/services/tracker/botBlocking/botEventQueue.ts
+++ b/server/src/services/tracker/botBlocking/botEventQueue.ts
@@ -79,6 +79,10 @@ class BotEventQueue {
detected_rate_anomaly: event.detectedRateAnomaly || false,
matched_ua_pattern: event.matchedUaPattern || "",
bot_category: event.botCategory || "",
+ bot_name: event.botName || "",
+ bot_operator: event.botOperator || "",
+ bot_purpose: event.botPurpose || "",
+ asn_provider: event.asnProvider || "",
client_bot_score: event.clientBotScore ?? null,
client_signal_mask: event.clientSignalMask ?? 0,
anomaly_reasons: event.anomalyReasons || "",
diff --git a/server/src/services/tracker/botBlocking/index.test.ts b/server/src/services/tracker/botBlocking/index.test.ts
index 77cee7756..0f893a635 100644
--- a/server/src/services/tracker/botBlocking/index.test.ts
+++ b/server/src/services/tracker/botBlocking/index.test.ts
@@ -72,7 +72,7 @@ describe("checkBotBlocking", () => {
}
});
- it("skips verified trusted server-side ingestion requests", async () => {
+ it("skips the browser-shaped layers for verified trusted server-side ingestion", async () => {
const result = await checkBotBlocking({
headers: {},
blockBots: true,
@@ -83,6 +83,59 @@ describe("checkBotBlocking", () => {
expect(result).toBeNull();
});
+ it("does not convict trusted server-side ingestion reporting a browser user agent", async () => {
+ // Empty headers and a datacenter IP would convict ordinary traffic on the
+ // header and ASN layers. Neither belongs to the visitor being reported.
+ const result = await checkBotBlocking({
+ headers: {},
+ blockBots: true,
+ trustedServerSideIngestion: true,
+ payload: { ...basePayload, userAgent: "Mozilla/5.0 Chrome/120 Safari/537.36" },
+ });
+
+ expect(result).toBeNull();
+ });
+
+ it("classifies a crawler user agent reported through trusted server-side ingestion", async () => {
+ const result = await checkBotBlocking({
+ headers: {},
+ blockBots: true,
+ trustedServerSideIngestion: true,
+ payload: {
+ ...basePayload,
+ userAgent: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.2; +https://openai.com/gptbot",
+ },
+ });
+
+ expect(result).toMatchObject({
+ isBot: true,
+ enforced: true,
+ detections: [{ layer: "ua_pattern" }],
+ eventProperties: {
+ detectedUaPattern: true,
+ detectedHeaderHeuristics: false,
+ detectedBotAsn: false,
+ detectedRateAnomaly: false,
+ botName: "GPTBot",
+ botOperator: "OpenAI",
+ botPurpose: "ai_training",
+ // The reporting server's address is not the bot's.
+ asnProvider: "",
+ },
+ });
+ });
+
+ it("leaves enforcement to blockBots for trusted server-side ingestion", async () => {
+ const result = await checkBotBlocking({
+ headers: {},
+ blockBots: false,
+ trustedServerSideIngestion: true,
+ payload: { ...basePayload, userAgent: "ClaudeBot/1.0" },
+ });
+
+ expect(result).toMatchObject({ isBot: true, enforced: false });
+ });
+
it("does not bypass bot blocking for an unverified bearer header", async () => {
const result = await checkBotBlocking({
headers: { authorization: "Bearer token" },
diff --git a/server/src/services/tracker/botBlocking/index.ts b/server/src/services/tracker/botBlocking/index.ts
index aa5a12c4f..0583e6385 100644
--- a/server/src/services/tracker/botBlocking/index.ts
+++ b/server/src/services/tracker/botBlocking/index.ts
@@ -73,6 +73,9 @@ export interface BotBlockingDetection {
layer: BotDetectionMethod;
botCategory?: string | null;
matchedPattern?: string | null;
+ botName?: string | null;
+ botOperator?: string | null;
+ botPurpose?: string | null;
reason?: string;
score?: number;
clientBotScore?: number;
@@ -99,6 +102,22 @@ export interface BotEventProperties {
detectedRateAnomaly: boolean;
matchedUaPattern: string;
botCategory: string;
+ /**
+ * Published identity of the bot, when the matched pattern is one of the
+ * curated ones. `matchedUaPattern` is the evidence (a regex source); these
+ * are the answer to "who is this and what do they want" β€” which is the only
+ * form in which crawl volume can be attributed to an operator or split into
+ * training, answer-engine indexing, and user-triggered agent fetches.
+ * Empty for a bot matched only by a generic upstream pattern.
+ */
+ botName: string;
+ botOperator: string;
+ botPurpose: string;
+ /**
+ * Curated provider behind the request's ASN. A user agent can claim any
+ * name, so this is the independent half of an attribution claim.
+ */
+ asnProvider: string;
clientBotScore?: number;
clientSignalMask?: number;
/**
@@ -135,6 +154,7 @@ function buildBotEventProperties(
const detectionLayers = new Set(detections.map(detection => detection.layer));
const uaDetection = detections.find(detection => detection.layer === "ua_pattern");
const anomalyDetection = detections.find(detection => detection.layer === "rate_anomaly");
+ const asnDetection = detections.find(detection => detection.layer === "bot_asn");
return {
isBot: true,
@@ -147,6 +167,10 @@ function buildBotEventProperties(
detectedRateAnomaly: detectionLayers.has("rate_anomaly"),
matchedUaPattern: uaDetection?.matchedPattern ?? "",
botCategory: uaDetection?.botCategory ?? "",
+ botName: uaDetection?.botName ?? "",
+ botOperator: uaDetection?.botOperator ?? "",
+ botPurpose: uaDetection?.botPurpose ?? "",
+ asnProvider: asnDetection?.asnProvider ?? "",
clientBotScore: clientSignalResult.scoreForStats,
clientSignalMask: clientSignalResult.maskForStats,
anomalyReasons: (anomalyDetection?.anomalyReasons ?? []).map(reason => reason.rule).join(","),
@@ -212,6 +236,52 @@ function getClientSignalResult(
};
}
+/**
+ * Bot detection for trusted server-side ingestion: the user-agent layer alone.
+ *
+ * The stale-browser-version sub-rule is left out too. It convicts a UA claiming
+ * Chrome older than the tracker that reported the hit β€” sound reasoning for a
+ * browser, meaningless for a server relaying someone else's user agent, and it
+ * would start convicting traffic that existing API users are sending today.
+ *
+ * Enforcement is not special-cased: the result goes back as an ordinary
+ * detection, so `blockBots` decides its fate exactly as it does for browser
+ * traffic and the row lands in `bot_events` or `bot_observations` accordingly.
+ */
+function classifyTrustedIngestion(
+ userAgent: string,
+ blockBots: boolean,
+ clientSignalResult: ReturnType<typeof getClientSignalResult>
+): BotDetectionResult | null {
+ const uaClassification = classifyUA(userAgent);
+ if (!uaClassification.isBot) {
+ return null;
+ }
+
+ const detections: BotBlockingDetection[] = [
+ {
+ layer: "ua_pattern",
+ botCategory: uaClassification.category,
+ matchedPattern: uaClassification.matchedPattern,
+ botName: uaClassification.name,
+ botOperator: uaClassification.operator,
+ botPurpose: uaClassification.purpose,
+ },
+ ];
+
+ recordBotDetections(["ua_pattern"], blockBots);
+
+ return {
+ isBot: true,
+ enforced: blockBots,
+ message: "Bot detected using ua-pattern",
+ detections,
+ // No ASN: the address belongs to the reporting server, not to the bot, so
+ // attributing it would be worse than leaving it empty.
+ eventProperties: buildBotEventProperties(detections, null, clientSignalResult),
+ };
+}
+
export async function checkBotBlocking({
headers,
blockBots,
@@ -230,16 +300,27 @@ export async function checkBotBlocking({
);
// Trusted server-side ingestion is authenticated first-party traffic that
- // reports its own IP and user agent: none of the layers below are meaningful
- // against it, and several would convict it outright. It is the only traffic
- // that skips detection.
+ // reports its own IP and user agent on someone else's behalf. Four of the
+ // five layers are meaningless against it and several would convict it
+ // outright: header heuristics read the reporting server's headers, client
+ // signals require a browser that never ran, the ASN belongs to whoever is
+ // doing the reporting, and rate anomaly would see one origin standing in for
+ // its entire audience.
+ //
+ // The UA layer is the exception, and the only way Rybbit can see a crawler
+ // that does not execute JavaScript. `POST /api/track` is the sole ingestion
+ // path, so a training crawler that fetches HTML and runs nothing is invisible
+ // unless the origin forwards it β€” and a forwarded `GPTBot` used to land in
+ // `events` as a human, because detection was skipped wholesale. Classifying
+ // the reported user agent routes it to `bot_events` with full identity
+ // instead.
//
// `blockBots` deliberately does NOT skip detection. It decides what happens to
// a detection, not whether one is looked for β€” a site that has turned blocking
// off used to get no evaluation at all, which left it with neither protection
// nor any record of what it was receiving.
if (trustedServerSideIngestion) {
- return null;
+ return classifyTrustedIngestion(userAgent, blockBots, clientSignalResult);
}
const detections: BotBlockingDetection[] = [];
@@ -264,6 +345,9 @@ export async function checkBotBlocking({
layer: "ua_pattern",
botCategory: uaClassification.category,
matchedPattern: uaClassification.matchedPattern,
+ botName: uaClassification.name,
+ botOperator: uaClassification.operator,
+ botPurpose: uaClassification.purpose,
});
} else {
// Layer 1b: a browser release too old to have run the tracker that
diff --git a/server/src/services/tracker/botBlocking/uaBots/index.test.ts b/server/src/services/tracker/botBlocking/uaBots/index.test.ts
index 1568f24b4..cfc6d15ba 100644
--- a/server/src/services/tracker/botBlocking/uaBots/index.test.ts
+++ b/server/src/services/tracker/botBlocking/uaBots/index.test.ts
@@ -125,6 +125,76 @@ describe("classifyUA", () => {
}
});
+ it("names AI bots and splits them by purpose", () => {
+ // `category: "ai"` covers a training crawler, an answer-engine indexer and
+ // a fetch a person asked for a second ago. Only `purpose` tells them apart,
+ // and the difference is the whole reason a site owner looks at this page.
+ const cases: [string, string, string, string][] = [
+ [
+ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.2; +https://openai.com/gptbot",
+ "GPTBot",
+ "OpenAI",
+ "ai_training",
+ ],
+ ["Mozilla/5.0 (compatible; ClaudeBot/1.0; [email protected])", "ClaudeBot", "Anthropic", "ai_training"],
+ ["CCBot/2.0 (https://commoncrawl.org/faq/)", "CCBot", "Common Crawl", "ai_training"],
+ ["Mozilla/5.0 (compatible; OAI-SearchBot/1.0; +https://openai.com/searchbot)", "OAI-SearchBot", "OpenAI", "ai_search"],
+ ["Mozilla/5.0 (compatible; PerplexityBot/1.0)", "PerplexityBot", "Perplexity", "ai_search"],
+ ["Mozilla/5.0 (compatible; ChatGPT-User/1.0; +https://openai.com/bot)", "ChatGPT-User", "OpenAI", "ai_agent"],
+ ["Mozilla/5.0 (compatible; Claude-User/1.0)", "Claude-User", "Anthropic", "ai_agent"],
+ ["Mozilla/5.0 (compatible; Perplexity-User/1.0)", "Perplexity-User", "Perplexity", "ai_agent"],
+ ["claude-code/1.0.0", "Claude Code", "Anthropic", "ai_agent"],
+ ["Google-Agent", "Google-Agent", "Google", "ai_agent"],
+ ];
+
+ for (const [ua, name, operator, purpose] of cases) {
+ const cls = classifyUA(ua);
+ expect(cls.isBot, ua).toBe(true);
+ expect(cls.category, ua).toBe("ai");
+ expect(cls.name, ua).toBe(name);
+ expect(cls.operator, ua).toBe(operator);
+ expect(cls.purpose, ua).toBe(purpose);
+ }
+ });
+
+ it("names bots outside the AI family too", () => {
+ expect(classifyUA("Mozilla/5.0 (compatible; AhrefsBot/7.0)")).toMatchObject({
+ name: "AhrefsBot",
+ operator: "Ahrefs",
+ purpose: "seo",
+ });
+ expect(classifyUA("facebookexternalhit/1.1")).toMatchObject({
+ name: "facebookexternalhit",
+ operator: "Meta",
+ purpose: "social_preview",
+ });
+ expect(classifyUA("python-requests/2.31.0")).toMatchObject({
+ name: "python-requests",
+ operator: null,
+ purpose: "scripted",
+ });
+ });
+
+ it("leaves identity null for a bot matched only by a generic upstream pattern", () => {
+ // The vendored isbot list is intentionally un-annotated: a hit on "crawl"
+ // or "spider" is a bot without a name, and inventing one from the regex
+ // would be worse than admitting we do not know.
+ const cls = classifyUA("Mozilla/5.0 (compatible; SomeUnknownCrawler/3.1)");
+ expect(cls.isBot).toBe(true);
+ expect(cls.name).toBeNull();
+ expect(cls.operator).toBeNull();
+ expect(cls.purpose).toBeNull();
+ });
+
+ it("returns null identity for a non-bot", () => {
+ expect(classifyUA("Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 Chrome/120 Safari/537.36")).toMatchObject({
+ isBot: false,
+ name: null,
+ operator: null,
+ purpose: null,
+ });
+ });
+
it("isBotUA boolean shorthand agrees with classifyUA", () => {
expect(isBotUA("Googlebot/2.1")).toBe(true);
expect(isBotUA("Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 Chrome/120 Safari/537.36")).toBe(false);
diff --git a/server/src/services/tracker/botBlocking/uaBots/index.ts b/server/src/services/tracker/botBlocking/uaBots/index.ts
index 28b3fab82..b3c5134d7 100644
--- a/server/src/services/tracker/botBlocking/uaBots/index.ts
+++ b/server/src/services/tracker/botBlocking/uaBots/index.ts
@@ -1,14 +1,29 @@
-import { ALL_BOT_PATTERNS, BotCategory } from "./patterns.js";
+import { ALL_BOT_PATTERNS, BotCategory, BotPurpose } from "./patterns.js";
-export type { BotCategory } from "./patterns.js";
+export type { BotCategory, BotPurpose } from "./patterns.js";
export interface BotClassification {
isBot: boolean;
category: BotCategory | null;
matchedPattern: string | null;
+ /**
+ * Human-readable identity, present only for the curated patterns. A match on
+ * a generic upstream rule ("crawl", "spider") is a bot without a name, and
+ * saying so is more useful than inventing one from the regex.
+ */
+ name: string | null;
+ operator: string | null;
+ purpose: BotPurpose | null;
}
-const NON_BOT: BotClassification = { isBot: false, category: null, matchedPattern: null };
+const NON_BOT: BotClassification = {
+ isBot: false,
+ category: null,
+ matchedPattern: null,
+ name: null,
+ operator: null,
+ purpose: null,
+};
// classifyUA runs on every event (when bot blocking is enabled) and, for bot
// user-agents, scans every compiled pattern. User-agent strings repeat heavily,
@@ -23,12 +38,21 @@ const COMBINED_REGEX = new RegExp(ALL_BOT_PATTERNS.map(p => p.pattern).join("|")
// Per-pattern compiled regexes for category lookup. Iterated in source order
// so first match wins β€” patterns are authored most-specific-first in patterns.ts.
-const COMPILED_PATTERNS: ReadonlyArray<{ regex: RegExp; category: BotCategory; pattern: string }> =
- ALL_BOT_PATTERNS.map(p => ({
- regex: new RegExp(p.pattern, "i"),
- category: p.category,
- pattern: p.pattern,
- }));
+const COMPILED_PATTERNS: ReadonlyArray<{
+ regex: RegExp;
+ category: BotCategory;
+ pattern: string;
+ name: string | null;
+ operator: string | null;
+ purpose: BotPurpose | null;
+}> = ALL_BOT_PATTERNS.map(p => ({
+ regex: new RegExp(p.pattern, "i"),
+ category: p.category,
+ pattern: p.pattern,
+ name: p.name ?? null,
+ operator: p.operator ?? null,
+ purpose: p.purpose ?? null,
+}));
/**
* Classify a user-agent string. Returns the first matching bot pattern, or
@@ -64,13 +88,13 @@ function computeClassification(userAgent: string): BotClassification {
if (!COMBINED_REGEX.test(userAgent)) {
return NON_BOT;
}
- for (const { regex, category, pattern } of COMPILED_PATTERNS) {
+ for (const { regex, category, pattern, name, operator, purpose } of COMPILED_PATTERNS) {
if (regex.test(userAgent)) {
- return { isBot: true, category, matchedPattern: pattern };
+ return { isBot: true, category, matchedPattern: pattern, name, operator, purpose };
}
}
// Combined regex matched but no individual pattern did β€” should be unreachable.
- return { isBot: true, category: "generic", matchedPattern: null };
+ return { isBot: true, category: "generic", matchedPattern: null, name: null, operator: null, purpose: null };
}
/**
diff --git a/server/src/services/tracker/botBlocking/uaBots/patterns.ts b/server/src/services/tracker/botBlocking/uaBots/patterns.ts
index ee8f1b29d..8b7f0764e 100644
--- a/server/src/services/tracker/botBlocking/uaBots/patterns.ts
+++ b/server/src/services/tracker/botBlocking/uaBots/patterns.ts
@@ -25,10 +25,43 @@ export type BotCategory =
| "headless" // headless browsers and browser automation
| "generic"; // matched a bot-ish pattern but uncategorized
+/**
+ * What the bot is *for*. `category` says which family a pattern belongs to;
+ * `purpose` says what the operator does with the fetch, which is the
+ * distinction site owners actually care about β€” a training crawler building a
+ * corpus, an answer engine indexing for retrieval, and a human asking an agent
+ * to open this page right now are three very different visitors that all share
+ * `category: "ai"`.
+ *
+ * Purpose is additive: `category` is what is persisted historically and what
+ * the existing Categories tab reads, so it keeps its meaning unchanged.
+ */
+export type BotPurpose =
+ | "ai_training" // corpus collection for model training
+ | "ai_search" // indexing for an AI answer engine
+ | "ai_agent" // a human asked an AI to fetch this page, right now
+ | "search" // classic search engine indexing
+ | "social_preview" // link unfurling
+ | "seo" // backlink / rank / site-audit crawlers
+ | "monitoring" // uptime, synthetic, performance
+ | "security" // scanners
+ | "scripted" // HTTP libraries and CLI clients
+ | "headless" // browser automation
+ | "unknown";
+
export interface BotPattern {
/** Regex source string. Compiled with the `i` flag. */
pattern: string;
category: BotCategory;
+ /**
+ * Human-readable bot name, e.g. "GPTBot". Only set on the curated
+ * EXTRA_BOT_PATTERNS entries β€” the vendored upstream list is matched by
+ * `pattern` string on re-sync, so it is deliberately left un-annotated.
+ */
+ name?: string;
+ /** Who operates the bot, e.g. "OpenAI". */
+ operator?: string;
+ purpose?: BotPurpose;
}
export const BOT_PATTERNS: BotPattern[] = [
@@ -221,79 +254,181 @@ export const BOT_PATTERNS: BotPattern[] = [
* Additional and priority AI / social / monitoring identifiers. Some duplicate
* upstream patterns intentionally so they win before broad generic rules like
* "(?<! cu)bots?" and "(?<!(?:lib))http".
+ *
+ * These are the only entries carrying `name` / `operator` / `purpose`. The
+ * upstream BOT_PATTERNS list above is re-synced by matching the `pattern`
+ * string against isbot's patterns.json, so annotating it would put that
+ * procedure at odds with itself; a named bot that matters belongs here instead,
+ * where it also wins the first-match race.
*/
export const EXTRA_BOT_PATTERNS: BotPattern[] = [
- // AI training / retrieval crawlers
- { pattern: "^openai/", category: "ai" },
- { pattern: "^claude-code/", category: "ai" },
- { pattern: "\\bgptbot\\b", category: "ai" },
- { pattern: "\\boai-searchbot\\b", category: "ai" },
- { pattern: "\\bchatgpt-user\\b", category: "ai" },
- { pattern: "\\bclaudebot\\b", category: "ai" },
- { pattern: "\\bclaude-user\\b", category: "ai" },
- { pattern: "\\bclaude-searchbot\\b", category: "ai" },
- { pattern: "\\bperplexitybot\\b", category: "ai" },
- { pattern: "\\bperplexity-user\\b", category: "ai" },
- { pattern: "\\bbytespider\\b", category: "ai" },
- { pattern: "\\bccbot\\b", category: "ai" },
- { pattern: "\\bgoogle-extended\\b", category: "ai" },
- { pattern: "\\bmeta-externalagent\\b", category: "ai" },
- { pattern: "\\bmeta-externalfetcher\\b", category: "ai" },
- { pattern: "\\bdiffbot\\b", category: "ai" },
- { pattern: "\\bcohere-ai\\b", category: "ai" },
- { pattern: "\\bapplebot-extended\\b", category: "ai" },
- { pattern: "\\bamazonbot\\b", category: "ai" },
- { pattern: "\\bmistralai-user\\b", category: "ai" },
- { pattern: "\\bgrokbot\\b", category: "ai" },
- { pattern: "\\byouchat\\b", category: "ai" },
+ // --- AI: training crawlers -------------------------------------------------
+ // Fetch pages to build a corpus. They do not send anyone back to the site.
+ { pattern: "\\bgptbot\\b", category: "ai", name: "GPTBot", operator: "OpenAI", purpose: "ai_training" },
+ { pattern: "\\bclaudebot\\b", category: "ai", name: "ClaudeBot", operator: "Anthropic", purpose: "ai_training" },
+ { pattern: "\\bccbot\\b", category: "ai", name: "CCBot", operator: "Common Crawl", purpose: "ai_training" },
+ { pattern: "\\bgoogle-extended\\b", category: "ai", name: "Google-Extended", operator: "Google", purpose: "ai_training" },
+ {
+ pattern: "\\bapplebot-extended\\b",
+ category: "ai",
+ name: "Applebot-Extended",
+ operator: "Apple",
+ purpose: "ai_training",
+ },
+ { pattern: "\\bbytespider\\b", category: "ai", name: "Bytespider", operator: "ByteDance", purpose: "ai_training" },
+ {
+ pattern: "\\bmeta-externalagent\\b",
+ category: "ai",
+ name: "Meta-ExternalAgent",
+ operator: "Meta",
+ purpose: "ai_training",
+ },
+ { pattern: "\\bamazonbot\\b", category: "ai", name: "Amazonbot", operator: "Amazon", purpose: "ai_training" },
+ { pattern: "\\bcohere-ai\\b", category: "ai", name: "cohere-ai", operator: "Cohere", purpose: "ai_training" },
+ { pattern: "\\bdiffbot\\b", category: "ai", name: "Diffbot", operator: "Diffbot", purpose: "ai_training" },
+ { pattern: "\\bomgilibot\\b", category: "ai", name: "Omgilibot", operator: "Webz.io", purpose: "ai_training" },
+
+ // --- AI: answer-engine indexers -------------------------------------------
+ // Crawl to build a retrieval index that an assistant cites. Unlike training
+ // crawlers these can send referral traffic back, which is why they are split
+ // out rather than folded into ai_training.
+ { pattern: "\\boai-searchbot\\b", category: "ai", name: "OAI-SearchBot", operator: "OpenAI", purpose: "ai_search" },
+ {
+ pattern: "\\bclaude-searchbot\\b",
+ category: "ai",
+ name: "Claude-SearchBot",
+ operator: "Anthropic",
+ purpose: "ai_search",
+ },
+ {
+ pattern: "\\bperplexitybot\\b",
+ category: "ai",
+ name: "PerplexityBot",
+ operator: "Perplexity",
+ purpose: "ai_search",
+ },
+ { pattern: "\\bduckassistbot\\b", category: "ai", name: "DuckAssistBot", operator: "DuckDuckGo", purpose: "ai_search" },
+ { pattern: "\\byouchat\\b", category: "ai", name: "YouChat", operator: "You.com", purpose: "ai_search" },
+ { pattern: "\\bgrokbot\\b", category: "ai", name: "GrokBot", operator: "xAI", purpose: "ai_search" },
+
+ // --- AI: user-triggered agents --------------------------------------------
+ // A person asked an assistant to open this page, so the fetch is one real
+ // human's intent rather than background crawling. Highest-value class to
+ // separate: these correlate with the AI referral channel, training crawls do
+ // not.
+ {
+ pattern: "\\bchatgpt-user\\b",
+ category: "ai",
+ name: "ChatGPT-User",
+ operator: "OpenAI",
+ purpose: "ai_agent",
+ },
+ { pattern: "\\bclaude-user\\b", category: "ai", name: "Claude-User", operator: "Anthropic", purpose: "ai_agent" },
+ {
+ pattern: "\\bperplexity-user\\b",
+ category: "ai",
+ name: "Perplexity-User",
+ operator: "Perplexity",
+ purpose: "ai_agent",
+ },
+ {
+ pattern: "\\bmistralai-user\\b",
+ category: "ai",
+ name: "MistralAI-User",
+ operator: "Mistral",
+ purpose: "ai_agent",
+ },
+ {
+ pattern: "\\bmeta-externalfetcher\\b",
+ category: "ai",
+ name: "Meta-ExternalFetcher",
+ operator: "Meta",
+ purpose: "ai_agent",
+ },
+ { pattern: "\\bgoogle-agent\\b", category: "ai", name: "Google-Agent", operator: "Google", purpose: "ai_agent" },
+ { pattern: "^openai/", category: "ai", name: "OpenAI API client", operator: "OpenAI", purpose: "ai_agent" },
+ { pattern: "^claude-code/", category: "ai", name: "Claude Code", operator: "Anthropic", purpose: "ai_agent" },
+ { pattern: "\\bcursor/", category: "ai", name: "Cursor", operator: "Cursor", purpose: "ai_agent" },
+ { pattern: "\\bmanus-user/", category: "ai", name: "Manus", operator: "Manus", purpose: "ai_agent" },
+ { pattern: "\\bfirecrawl\\b", category: "ai", name: "Firecrawl", operator: "Firecrawl", purpose: "ai_agent" },
// Search crawlers whose names can be swallowed by generic "*bot*" rules
- { pattern: "yandexbot", category: "search" },
- { pattern: "duckduckgo", category: "search" },
- { pattern: "slurp", category: "search" },
+ { pattern: "yandexbot", category: "search", name: "YandexBot", operator: "Yandex", purpose: "search" },
+ { pattern: "duckduckgo", category: "search", name: "DuckDuckBot", operator: "DuckDuckGo", purpose: "search" },
+ { pattern: "slurp", category: "search", name: "Yahoo! Slurp", operator: "Yahoo", purpose: "search" },
// Social link previewers not caught by upstream
- { pattern: "facebookexternalhit", category: "social" },
- { pattern: "facebot", category: "social" },
- { pattern: "twitterbot", category: "social" },
- { pattern: "slackbot", category: "social" },
- { pattern: "discordbot", category: "social" },
- { pattern: "linkedinbot", category: "social" },
- { pattern: "telegrambot", category: "social" },
- { pattern: "skypeuripreview", category: "social" },
- { pattern: "redditbot", category: "social" },
- { pattern: "pinterestbot", category: "social" },
- { pattern: "embedly", category: "social" },
+ {
+ pattern: "facebookexternalhit",
+ category: "social",
+ name: "facebookexternalhit",
+ operator: "Meta",
+ purpose: "social_preview",
+ },
+ { pattern: "facebot", category: "social", name: "Facebot", operator: "Meta", purpose: "social_preview" },
+ { pattern: "twitterbot", category: "social", name: "Twitterbot", operator: "X", purpose: "social_preview" },
+ { pattern: "slackbot", category: "social", name: "Slackbot", operator: "Slack", purpose: "social_preview" },
+ { pattern: "discordbot", category: "social", name: "Discordbot", operator: "Discord", purpose: "social_preview" },
+ { pattern: "linkedinbot", category: "social", name: "LinkedInBot", operator: "LinkedIn", purpose: "social_preview" },
+ { pattern: "telegrambot", category: "social", name: "TelegramBot", operator: "Telegram", purpose: "social_preview" },
+ {
+ pattern: "skypeuripreview",
+ category: "social",
+ name: "SkypeUriPreview",
+ operator: "Microsoft",
+ purpose: "social_preview",
+ },
+ { pattern: "redditbot", category: "social", name: "RedditBot", operator: "Reddit", purpose: "social_preview" },
+ { pattern: "pinterestbot", category: "social", name: "Pinterestbot", operator: "Pinterest", purpose: "social_preview" },
+ { pattern: "embedly", category: "social", name: "Embedly", operator: "Embedly", purpose: "social_preview" },
// SEO crawlers commonly seen in the wild
- { pattern: "ahrefsbot", category: "seo" },
- { pattern: "semrushbot", category: "seo" },
- { pattern: "mj12bot", category: "seo" },
- { pattern: "dotbot", category: "seo" },
- { pattern: "rogerbot", category: "seo" },
- { pattern: "screaming frog seo spider", category: "seo" },
- { pattern: "serpstatbot", category: "seo" },
+ { pattern: "ahrefsbot", category: "seo", name: "AhrefsBot", operator: "Ahrefs", purpose: "seo" },
+ { pattern: "semrushbot", category: "seo", name: "SemrushBot", operator: "Semrush", purpose: "seo" },
+ { pattern: "mj12bot", category: "seo", name: "MJ12bot", operator: "Majestic", purpose: "seo" },
+ { pattern: "dotbot", category: "seo", name: "DotBot", operator: "Moz", purpose: "seo" },
+ { pattern: "rogerbot", category: "seo", name: "rogerbot", operator: "Moz", purpose: "seo" },
+ {
+ pattern: "screaming frog seo spider",
+ category: "seo",
+ name: "Screaming Frog",
+ operator: "Screaming Frog",
+ purpose: "seo",
+ },
+ { pattern: "serpstatbot", category: "seo", name: "serpstatbot", operator: "Serpstat", purpose: "seo" },
// Scripted clients and browser automation
- { pattern: "python-requests", category: "framework" },
- { pattern: "curl/", category: "framework" },
- { pattern: "^wget", category: "framework" },
- { pattern: "postmanruntime", category: "framework" },
- { pattern: "apache-httpclient", category: "framework" },
- { pattern: "headlesschrome", category: "headless" },
- { pattern: "phantomjs", category: "headless" },
- { pattern: "\\bplaywright\\b", category: "headless" },
- { pattern: "\\bselenium\\b", category: "headless" },
+ { pattern: "python-requests", category: "framework", name: "python-requests", purpose: "scripted" },
+ { pattern: "curl/", category: "framework", name: "curl", purpose: "scripted" },
+ { pattern: "^wget", category: "framework", name: "Wget", purpose: "scripted" },
+ { pattern: "postmanruntime", category: "framework", name: "Postman", operator: "Postman", purpose: "scripted" },
+ { pattern: "apache-httpclient", category: "framework", name: "Apache HttpClient", purpose: "scripted" },
+ { pattern: "headlesschrome", category: "headless", name: "HeadlessChrome", purpose: "headless" },
+ { pattern: "phantomjs", category: "headless", name: "PhantomJS", purpose: "headless" },
+ { pattern: "\\bplaywright\\b", category: "headless", name: "Playwright", purpose: "headless" },
+ { pattern: "\\bselenium\\b", category: "headless", name: "Selenium", purpose: "headless" },
// Monitoring services
- { pattern: "pingdom", category: "monitoring" },
- { pattern: "uptimerobot", category: "monitoring" },
- { pattern: "datadog", category: "monitoring" },
- { pattern: "newrelic", category: "monitoring" },
- { pattern: "site24x7", category: "monitoring" },
- { pattern: "betteruptime", category: "monitoring" },
- { pattern: "statuscake", category: "monitoring" },
- { pattern: "chrome-lighthouse", category: "monitoring" },
+ { pattern: "pingdom", category: "monitoring", name: "Pingdom", operator: "Pingdom", purpose: "monitoring" },
+ { pattern: "uptimerobot", category: "monitoring", name: "UptimeRobot", operator: "UptimeRobot", purpose: "monitoring" },
+ { pattern: "datadog", category: "monitoring", name: "Datadog", operator: "Datadog", purpose: "monitoring" },
+ { pattern: "newrelic", category: "monitoring", name: "New Relic", operator: "New Relic", purpose: "monitoring" },
+ { pattern: "site24x7", category: "monitoring", name: "Site24x7", operator: "Site24x7", purpose: "monitoring" },
+ {
+ pattern: "betteruptime",
+ category: "monitoring",
+ name: "Better Uptime",
+ operator: "Better Stack",
+ purpose: "monitoring",
+ },
+ { pattern: "statuscake", category: "monitoring", name: "StatusCake", operator: "StatusCake", purpose: "monitoring" },
+ {
+ pattern: "chrome-lighthouse",
+ category: "monitoring",
+ name: "Lighthouse",
+ operator: "Google",
+ purpose: "monitoring",
+ },
];
// EXTRA patterns come first so explicit named bots (ClaudeBot, AhrefsBot, etc.)
diff --git a/server/src/services/tracker/const.ts b/server/src/services/tracker/const.ts
index 67a6eaff2..47faab4f0 100644
--- a/server/src/services/tracker/const.ts
+++ b/server/src/services/tracker/const.ts
@@ -1,3 +1,5 @@
+import { AI_CHAT_DOMAINS } from "@rybbit/shared";
+
const searchDomains = [
// Google and variants
"google.com",
@@ -294,35 +296,10 @@ const shoppingDomains = [
"foodpanda.com",
];
-// AI chat domains
-const aiChatDomains = [
- "chatgpt.com",
- "chat.openai.com",
- "claude.ai",
- "gemini.google.com",
- "copilot.microsoft.com",
- "deepseek.com",
- "chat.deepseek.com",
- "chat.mistral.ai",
- "mistral.ai",
- "meta.ai",
- "poe.com",
- "grok.com",
- "pi.ai",
- "heypi.com",
- "coral.cohere.com",
- "character.ai",
- "qwen.ai",
- "jasper.ai",
- "writesonic.com",
- "chatsonic.com",
- "perplexity.ai",
- "you.com",
- "phind.com",
- "andi.com",
- "cursor.com",
- "codeium.com",
-];
+// AI chat domains. Sourced from shared/ so that channel classification and the
+// bots dashboard's crawl-vs-referral comparison cannot drift apart β€” the
+// dashboard has to spell an operator's name the same way the bot patterns do.
+const aiChatDomains = AI_CHAT_DOMAINS;
// AI chat sources (utm_source values)
const aiChatSources = [
diff --git a/shared/src/aiOperators.ts b/shared/src/aiOperators.ts
new file mode 100644
index 000000000..141c85806
--- /dev/null
+++ b/shared/src/aiOperators.ts
@@ -0,0 +1,73 @@
+/**
+ * AI operators, and the referrer domains that belong to each.
+ *
+ * This is the join key between the two halves of a site's AI story, which are
+ * collected by completely different code paths and would otherwise never meet:
+ *
+ * - **Crawls** β€” `bot_events.bot_operator`, written during bot detection from
+ * the curated user-agent patterns.
+ * - **Referrals** β€” `events` rows whose `channel` is `"AI"`, classified from the
+ * referrer domain.
+ *
+ * Put next to each other they answer the question a site owner actually has β€”
+ * "this company read 4,000 of my pages; did it send anyone back?" β€” so both
+ * sides have to spell an operator's name identically. That is why the mapping
+ * lives here rather than being written out twice.
+ *
+ * The `operator` values must match the `operator` field on the curated bot
+ * patterns in `server/src/services/tracker/botBlocking/uaBots/patterns.ts`.
+ */
+export const AI_OPERATOR_REFERRER_DOMAINS: Record<string, string[]> = {
+ OpenAI: ["chatgpt.com", "chat.openai.com"],
+ Anthropic: ["claude.ai"],
+ Google: ["gemini.google.com"],
+ Microsoft: ["copilot.microsoft.com"],
+ Perplexity: ["perplexity.ai"],
+ Meta: ["meta.ai"],
+ Mistral: ["chat.mistral.ai", "mistral.ai"],
+ xAI: ["grok.com"],
+ "You.com": ["you.com"],
+ // DuckDuckGo is deliberately absent: DuckAssistBot crawls, but a referral
+ // from duckduckgo.com is organic search, not an AI chat hand-off, and
+ // listing it here would reclassify every DuckDuckGo visit as AI traffic.
+ Cursor: ["cursor.com"],
+ Cohere: ["coral.cohere.com"],
+};
+
+/**
+ * Referrer domain β†’ operator, flattened. Lookups are by the domain form the
+ * events table stores (`domainWithoutWWW(referrer)`).
+ */
+export const AI_REFERRER_DOMAIN_TO_OPERATOR: Record<string, string> = Object.fromEntries(
+ Object.entries(AI_OPERATOR_REFERRER_DOMAINS).flatMap(([operator, domains]) =>
+ domains.map(domain => [domain, operator] as const)
+ )
+);
+
+/**
+ * AI chat domains that have no crawler counterpart in the bot patterns. They
+ * still belong to the AI acquisition channel β€” someone arrived from them β€” but
+ * there is no operator to attribute crawls to, so they are kept apart from the
+ * mapping above rather than given a made-up operator name.
+ */
+export const AI_CHAT_ONLY_DOMAINS: string[] = [
+ "deepseek.com",
+ "chat.deepseek.com",
+ "poe.com",
+ "pi.ai",
+ "heypi.com",
+ "character.ai",
+ "qwen.ai",
+ "jasper.ai",
+ "writesonic.com",
+ "chatsonic.com",
+ "phind.com",
+ "andi.com",
+ "codeium.com",
+];
+
+/** Every domain that counts as AI chat traffic, for channel classification. */
+export const AI_CHAT_DOMAINS: string[] = [
+ ...Object.keys(AI_REFERRER_DOMAIN_TO_OPERATOR),
+ ...AI_CHAT_ONLY_DOMAINS,
+];
diff --git a/shared/src/index.ts b/shared/src/index.ts
index 51cdce44d..f90923a89 100644
--- a/shared/src/index.ts
+++ b/shared/src/index.ts
@@ -5,3 +5,4 @@ export * from "./params";
export * from "./scopes";
export * from "./time";
export * from "./performance";
+export * from "./aiOperators";