Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Persist per-event ASN for identity-splitting diagnostics

Bill Yangb7b1065062c7
diff --git a/server/src/db/clickhouse/schema/core.ts b/server/src/db/clickhouse/schema/core.ts
index a05b75db1..9d2be85be 100644
--- a/server/src/db/clickhouse/schema/core.ts
+++ b/server/src/db/clickhouse/schema/core.ts
@@ -17,6 +17,9 @@ const EVENTS_COLUMNS_TO_ENSURE: ColumnDefinition[] = [
{ name: "import_id", definition: "import_id Nullable(UUID)" },
{ name: "tag", definition: "tag LowCardinality(String) DEFAULT ''" },
{ name: "feature_flags", definition: "feature_flags Map(String, String) DEFAULT map()" },
+ { name: "asn", definition: "asn Nullable(UInt32)" },
+ { name: "asn_org", definition: "asn_org LowCardinality(String) DEFAULT ''" },
+ { name: "is_datacenter_asn", definition: "is_datacenter_asn UInt8 DEFAULT 0" },
];
async function ensureEventsColumns() {
@@ -82,7 +85,10 @@ export async function initializeCoreTables() {
identified_user_id String DEFAULT '',
import_id Nullable(UUID),
tag LowCardinality(String) DEFAULT '',
- feature_flags Map(String, String) DEFAULT map()
+ feature_flags Map(String, String) DEFAULT map(),
+ asn Nullable(UInt32),
+ asn_org LowCardinality(String) DEFAULT '',
+ is_datacenter_asn UInt8 DEFAULT 0
)
ENGINE = MergeTree()
PARTITION BY toYYYYMM(timestamp)
diff --git a/server/src/services/tracker/pageviewQueue.test.ts b/server/src/services/tracker/pageviewQueue.test.ts
new file mode 100644
index 000000000..6d0595bbf
--- /dev/null
+++ b/server/src/services/tracker/pageviewQueue.test.ts
@@ -0,0 +1,98 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import type UAParser from "ua-parser-js";
+import type { TotalTrackingPayload } from "./utils.js";
+
+const mocks = vi.hoisted(() => ({
+ insert: vi.fn(),
+ getLocation: vi.fn(),
+ lookupAsn: vi.fn(),
+ isDatacenterAsn: vi.fn(),
+}));
+
+vi.mock("../../db/clickhouse/clickhouse.js", () => ({
+ clickhouse: { insert: mocks.insert },
+}));
+
+vi.mock("../../db/geolocation/geolocation.js", () => ({
+ getLocation: mocks.getLocation,
+}));
+
+vi.mock("../../db/geolocation/asn.js", () => ({
+ lookupAsn: mocks.lookupAsn,
+}));
+
+vi.mock("./botBlocking/datacenterAsns.js", () => ({
+ isDatacenterAsn: mocks.isDatacenterAsn,
+}));
+
+vi.useFakeTimers();
+const { pageviewQueue } = await import("./pageviewQueue.js");
+
+function makePayload(overrides: Partial<TotalTrackingPayload & { sessionId: string }> = {}) {
+ return {
+ site_id: 42,
+ hostname: "example.com",
+ pathname: "/",
+ querystring: "",
+ screenWidth: 1920,
+ screenHeight: 1080,
+ language: "en",
+ page_title: "",
+ referrer: "",
+ type: "pageview",
+ ipAddress: "203.0.113.10",
+ timestamp: new Date().toISOString(),
+ ua: { browser: {}, os: {} } as UAParser.IResult,
+ userAgent: "Mozilla/5.0",
+ userId: "fingerprint-abc",
+ identifiedUserId: "",
+ sessionId: "session-abc",
+ ...overrides,
+ } as TotalTrackingPayload & { sessionId: string };
+}
+
+async function flush() {
+ await pageviewQueue.add(makePayload());
+ await vi.advanceTimersByTimeAsync(1000);
+}
+
+describe("pageviewQueue ASN enrichment", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ mocks.getLocation.mockResolvedValue({});
+ mocks.insert.mockResolvedValue(undefined);
+ });
+
+ it("stores the ASN, org, and datacenter classification for a recognized IP", async () => {
+ mocks.lookupAsn.mockReturnValue({ asn: 16509, organization: "AMAZON-02" });
+ mocks.isDatacenterAsn.mockReturnValue(true);
+
+ await flush();
+
+ const row = mocks.insert.mock.calls[0][0].values[0];
+ expect(row.asn).toBe(16509);
+ expect(row.asn_org).toBe("AMAZON-02");
+ expect(row.is_datacenter_asn).toBe(1);
+ });
+
+ it("defaults to null/empty/0 when the IP has no ASN match", async () => {
+ mocks.lookupAsn.mockReturnValue(null);
+
+ await flush();
+
+ const row = mocks.insert.mock.calls[0][0].values[0];
+ expect(row.asn).toBeNull();
+ expect(row.asn_org).toBe("");
+ expect(row.is_datacenter_asn).toBe(0);
+ });
+
+ it("marks residential ASNs as non-datacenter", async () => {
+ mocks.lookupAsn.mockReturnValue({ asn: 7922, organization: "COMCAST-7922" });
+ mocks.isDatacenterAsn.mockReturnValue(false);
+
+ await flush();
+
+ const row = mocks.insert.mock.calls[0][0].values[0];
+ expect(row.is_datacenter_asn).toBe(0);
+ });
+});
diff --git a/server/src/services/tracker/pageviewQueue.ts b/server/src/services/tracker/pageviewQueue.ts
index 1d9d71b12..259f4128a 100644
--- a/server/src/services/tracker/pageviewQueue.ts
+++ b/server/src/services/tracker/pageviewQueue.ts
@@ -1,8 +1,10 @@
import { DateTime } from "luxon";
import { clickhouse } from "../../db/clickhouse/clickhouse.js";
+import { lookupAsn } from "../../db/geolocation/asn.js";
import { getLocation } from "../../db/geolocation/geolocation.js";
import { createServiceLogger } from "../../lib/logger/logger.js";
import { getDeviceType } from "../../utils.js";
+import { isDatacenterAsn } from "./botBlocking/datacenterAsns.js";
import { getChannel } from "./getChannel.js";
import { clearSelfReferrer, getAllUrlParams, TotalTrackingPayload } from "./utils.js";
@@ -63,6 +65,12 @@ class PageviewQueue {
const city = dataForIp?.city || "";
const timezone = dataForIp?.timeZone || "";
+ // Same MaxMind lookup already used to decide identity bucketing
+ // (bucketIpForIdentity) and bot-detection eligibility; stored here purely
+ // for debugging identity-splitting reports, treated like geo data
+ // (independent of storeIp) since it identifies a network, not a device.
+ const asnInfo = lookupAsn(pv.ipAddress);
+
// Check if referrer is from the same domain and clear it if so
let referrer = clearSelfReferrer(pv.referrer || "", pv.hostname || "");
@@ -112,6 +120,9 @@ class PageviewQueue {
tag: pv.tag || "",
feature_flags: pv.feature_flags || {},
import_id: null,
+ asn: asnInfo?.asn ?? null,
+ asn_org: asnInfo?.organization || "",
+ is_datacenter_asn: asnInfo && isDatacenterAsn(asnInfo.asn) ? 1 : 0,
};
});