🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
import { clickhouse } from "../../db/clickhouse/clickhouse.js";
import { toClickHouseDateTime } from "../../db/clickhouse/dateTime.js";
import { lookupAsn } from "../../db/geolocation/asn.js";
import { getLocation } from "../../db/geolocation/geolocation.js";
import { createServiceLogger } from "../../lib/logger/logger.js";
import { getDeviceType } from "../../utils.js";
import { isDatacenterAsn } from "./botBlocking/datacenterAsns.js";
import { getChannel } from "./getChannel.js";
import {
clearSelfReferrer,
EVENTS_INSERT_SETTINGS,
getAllUrlParams,
parseEventProperties,
TotalTrackingPayload,
} from "./utils.js";
type TotalPayload = TotalTrackingPayload & {
sessionId: string;
};
const PAGEVIEW_BATCH_SIZE = 5000;
const PAGEVIEW_FLUSH_INTERVAL_MS = 1000;
class PageviewQueue {
private queue: TotalPayload[] = [];
private batchSize = PAGEVIEW_BATCH_SIZE;
private interval = PAGEVIEW_FLUSH_INTERVAL_MS;
private processing = false;
private logger = createServiceLogger("pageview-queue");
constructor() {
// Start processing interval
setInterval(() => this.processQueue(), this.interval);
}
async add(pageview: TotalPayload) {
this.queue.push(pageview);
}
/**
* Flushes everything still buffered. Only for shutdown: process exit would
* otherwise drop up to a flush interval of events on every deploy.
*/
async drain(): Promise<void> {
while (this.processing || this.queue.length > 0) {
if (this.processing) {
await new Promise(resolve => setTimeout(resolve, 50));
} else {
await this.processQueue();
}
}
}
private async processQueue() {
if (this.processing || this.queue.length === 0) return;
this.processing = true;
// Get batch of pageviews
const batch = this.queue.splice(0, this.batchSize);
try {
const ips = [...new Set(batch.map(pv => pv.ipAddress))];
const geoData = await getLocation(ips);
// Process each pageview with its geo data
const processedPageviews = batch
.filter(pv => {
if (pv.site_id == 9133 && pv.screenWidth == 800 && pv.screenHeight == 600) {
return false;
}
return true;
})
.map(pv => {
const dataForIp = geoData?.[pv.ipAddress];
const countryCode = dataForIp?.countryIso || "";
const regionCode = dataForIp?.region || "";
const latitude = dataForIp?.latitude || 0;
const longitude = dataForIp?.longitude || 0;
const city = dataForIp?.city || "";
const timezone = dataForIp?.timeZone || "";
// Same MaxMind lookup already used to decide identity bucketing
// (bucketIpForIdentity) and bot-detection eligibility; stored here purely
// for debugging identity-splitting reports, treated like geo data
// (independent of storeIp) since it identifies a network, not a device.
const asnInfo = lookupAsn(pv.ipAddress);
// Check if referrer is from the same domain and clear it if so
let referrer = clearSelfReferrer(pv.referrer || "", pv.hostname || "");
// Get all URL parameters for the url_parameters map
const allUrlParams = getAllUrlParams(pv.querystring || "");
const receivedAt = toClickHouseDateTime(pv.timestamp);
return {
site_id: pv.site_id,
// Keep the original DateTime column for the MergeTree key and
// whole-second time filters; timestamp_ms preserves arrival order
// for funnels and entry/exit attribution.
// Both get the same explicit-UTC instant; the DateTime column
// truncates to the second on its own.
timestamp: receivedAt,
timestamp_ms: receivedAt,
session_id: pv.sessionId,
user_id: pv.userId, // Always the device fingerprint
identified_user_id: pv.identifiedUserId || "", // Custom user ID when identified
hostname: pv.hostname || "",
pathname: pv.pathname || "",
querystring: pv.querystring || "",
page_title: pv.page_title || "",
referrer: referrer,
channel: getChannel(referrer, pv.querystring, pv.hostname),
browser: pv.ua.browser.name || "",
browser_version: pv.ua.browser.major || "",
operating_system: pv.ua.os.name || "",
operating_system_version: pv.ua.os.version || "",
language: pv.language || "",
screen_width: pv.screenWidth || 0,
screen_height: pv.screenHeight || 0,
device_type: getDeviceType(pv.screenWidth, pv.screenHeight, pv.ua),
country: countryCode,
region: countryCode && regionCode ? countryCode + "-" + regionCode : "",
city: city || "",
lat: latitude || 0,
lon: longitude || 0,
type: pv.type || "pageview",
event_name: pv.event_name || "",
props: parseEventProperties(pv.properties),
url_parameters: allUrlParams,
// Performance metrics (only included for performance events).
// ?? not ||: 0 is a legitimate measurement (a perfect CLS score is 0)
// and must not be coerced to NULL, which would skew percentiles.
lcp: pv.lcp ?? null,
cls: pv.cls ?? null,
inp: pv.inp ?? null,
fcp: pv.fcp ?? null,
ttfb: pv.ttfb ?? null,
ip: pv.storeIp ? pv.ipAddress : null,
timezone: timezone,
tag: pv.tag || "",
feature_flags: pv.feature_flags || {},
visitor_id: pv.visitor_id || "",
import_id: null,
asn: asnInfo?.asn ?? null,
asn_org: asnInfo?.organization || "",
is_datacenter_asn: asnInfo && isDatacenterAsn(asnInfo.asn) ? 1 : 0,
};
});
// this.logger.info({ count: processedPageviews.length }, "Bulk insert to ClickHouse");
// Bulk insert into database
await clickhouse.insert({
table: "events",
values: processedPageviews,
format: "JSONEachRow",
clickhouse_settings: EVENTS_INSERT_SETTINGS,
});
} catch (error) {
this.logger.error({ err: error }, "Error processing pageview queue");
} finally {
this.processing = false;
}
}
}
// Create singleton instance
export const pageviewQueue = new PageviewQueue();