🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
import { clickhouse } from "../../../db/clickhouse/clickhouse.js";
import { toClickHouseDateTime } from "../../../db/clickhouse/dateTime.js";
import { getLocation } from "../../../db/geolocation/geolocation.js";
import { createServiceLogger } from "../../../lib/logger/logger.js";
import { getDeviceType } from "../../../utils.js";
import { clearSelfReferrer, type TotalTrackingPayload } from "../utils.js";
import type { BotEventProperties } from "./index.js";
type BotEventPayload = TotalTrackingPayload &
BotEventProperties & {
sessionId: string;
};
const BOT_EVENT_BATCH_SIZE = 5000;
const BOT_EVENT_FLUSH_INTERVAL_MS = 1000;
class BotEventQueue {
private queue: BotEventPayload[] = [];
private batchSize = BOT_EVENT_BATCH_SIZE;
private interval = BOT_EVENT_FLUSH_INTERVAL_MS;
private processing = false;
private table: string;
private logger: ReturnType<typeof createServiceLogger>;
constructor(table: string, loggerName: string) {
this.table = table;
this.logger = createServiceLogger(loggerName);
setInterval(() => this.processQueue(), this.interval);
}
async add(botEvent: BotEventPayload) {
this.queue.push(botEvent);
}
/**
* Flushes everything still buffered. Only for shutdown: process exit would
* otherwise drop up to a flush interval of events on every deploy.
*/
async drain(): Promise<void> {
while (this.processing || this.queue.length > 0) {
if (this.processing) {
await new Promise(resolve => setTimeout(resolve, 50));
} else {
await this.processQueue();
}
}
}
private async processQueue() {
if (this.processing || this.queue.length === 0) return;
this.processing = true;
const batch = this.queue.splice(0, this.batchSize);
try {
const ips = [...new Set(batch.map(event => event.ipAddress))];
const geoData = await getLocation(ips);
const processedBotEvents = batch.map(event => {
const dataForIp = geoData?.[event.ipAddress];
const countryCode = dataForIp?.countryIso || "";
const regionCode = dataForIp?.region || "";
const referrer = clearSelfReferrer(event.referrer || "", event.hostname || "");
return {
site_id: event.site_id,
timestamp: toClickHouseDateTime(event.timestamp),
session_id: event.sessionId,
user_id: event.userId,
hostname: event.hostname || "",
pathname: event.pathname || "",
querystring: event.querystring || "",
referrer,
browser: event.ua.browser.name || "",
browser_version: event.ua.browser.major || "",
operating_system: event.ua.os.name || "",
operating_system_version: event.ua.os.version || "",
country: countryCode,
region: countryCode && regionCode ? countryCode + "-" + regionCode : "",
city: dataForIp?.city || "",
lat: dataForIp?.latitude || 0,
lon: dataForIp?.longitude || 0,
screen_width: event.screenWidth || 0,
screen_height: event.screenHeight || 0,
device_type: getDeviceType(event.screenWidth, event.screenHeight, event.ua),
type: event.type || "pageview",
asn: event.botAsn ?? null,
asn_org: event.botAsnOrg || "",
detected_ua_pattern: event.detectedUaPattern || false,
detected_header_heuristics: event.detectedHeaderHeuristics || false,
detected_client_signals: event.detectedClientSignals || false,
detected_bot_asn: event.detectedBotAsn || false,
detected_rate_anomaly: event.detectedRateAnomaly || false,
matched_ua_pattern: event.matchedUaPattern || "",
bot_category: event.botCategory || "",
bot_name: event.botName || "",
bot_operator: event.botOperator || "",
bot_purpose: event.botPurpose || "",
asn_provider: event.asnProvider || "",
client_bot_score: event.clientBotScore ?? null,
client_signal_mask: event.clientSignalMask ?? 0,
anomaly_reasons: event.anomalyReasons || "",
anomaly_score: event.anomalyScore ?? 0,
};
});
await clickhouse.insert({
table: this.table,
values: processedBotEvents,
format: "JSONEachRow",
});
} catch (error) {
this.logger.error({ err: error }, "Error processing bot event queue");
} finally {
this.processing = false;
}
}
}
export const botEventQueue = new BotEventQueue("bot_events", "bot-event-queue");
// Detections that were NOT enforced: on a site with bot blocking disabled the
// event is tracked as normal, so this row is the only evidence that detection
// fired. The bot_observations schema deliberately mirrors bot_events
// column-for-column so the same analysis queries work against either table.
export const botObservationQueue = new BotEventQueue("bot_observations", "bot-observation-queue");