🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Persist per-event ASN for identity-splitting diagnostics
Bill Yangb7b1065062c7
diff --git a/server/src/db/clickhouse/schema/core.ts b/server/src/db/clickhouse/schema/core.tsindex a05b75db1..9d2be85be 100644--- a/server/src/db/clickhouse/schema/core.ts+++ b/server/src/db/clickhouse/schema/core.ts@@ -17,6 +17,9 @@ const EVENTS_COLUMNS_TO_ENSURE: ColumnDefinition[] = [{ name: "import_id", definition: "import_id Nullable(UUID)" },{ name: "tag", definition: "tag LowCardinality(String) DEFAULT ''" },{ name: "feature_flags", definition: "feature_flags Map(String, String) DEFAULT map()" },+ { name: "asn", definition: "asn Nullable(UInt32)" },+ { name: "asn_org", definition: "asn_org LowCardinality(String) DEFAULT ''" },+ { name: "is_datacenter_asn", definition: "is_datacenter_asn UInt8 DEFAULT 0" },];async function ensureEventsColumns() {@@ -82,7 +85,10 @@ export async function initializeCoreTables() {identified_user_id String DEFAULT '',import_id Nullable(UUID),tag LowCardinality(String) DEFAULT '',- feature_flags Map(String, String) DEFAULT map()+ feature_flags Map(String, String) DEFAULT map(),+ asn Nullable(UInt32),+ asn_org LowCardinality(String) DEFAULT '',+ is_datacenter_asn UInt8 DEFAULT 0)ENGINE = MergeTree()PARTITION BY toYYYYMM(timestamp)diff --git a/server/src/services/tracker/pageviewQueue.test.ts b/server/src/services/tracker/pageviewQueue.test.tsnew file mode 100644index 000000000..6d0595bbf--- /dev/null+++ b/server/src/services/tracker/pageviewQueue.test.ts@@ -0,0 +1,98 @@+import { beforeEach, describe, expect, it, vi } from "vitest";+import type UAParser from "ua-parser-js";+import type { TotalTrackingPayload } from "./utils.js";++const mocks = vi.hoisted(() => ({+ insert: vi.fn(),+ getLocation: vi.fn(),+ lookupAsn: vi.fn(),+ isDatacenterAsn: vi.fn(),+}));++vi.mock("../../db/clickhouse/clickhouse.js", () => ({+ clickhouse: { insert: mocks.insert },+}));++vi.mock("../../db/geolocation/geolocation.js", () => ({+ getLocation: mocks.getLocation,+}));++vi.mock("../../db/geolocation/asn.js", () => ({+ lookupAsn: mocks.lookupAsn,+}));++vi.mock("./botBlocking/datacenterAsns.js", () => ({+ isDatacenterAsn: mocks.isDatacenterAsn,+}));++vi.useFakeTimers();+const { pageviewQueue } = await import("./pageviewQueue.js");++function makePayload(overrides: Partial<TotalTrackingPayload & { sessionId: string }> = {}) {+ return {+ site_id: 42,+ hostname: "example.com",+ pathname: "/",+ querystring: "",+ screenWidth: 1920,+ screenHeight: 1080,+ language: "en",+ page_title: "",+ referrer: "",+ type: "pageview",+ ipAddress: "203.0.113.10",+ timestamp: new Date().toISOString(),+ ua: { browser: {}, os: {} } as UAParser.IResult,+ userAgent: "Mozilla/5.0",+ userId: "fingerprint-abc",+ identifiedUserId: "",+ sessionId: "session-abc",+ ...overrides,+ } as TotalTrackingPayload & { sessionId: string };+}++async function flush() {+ await pageviewQueue.add(makePayload());+ await vi.advanceTimersByTimeAsync(1000);+}++describe("pageviewQueue ASN enrichment", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ mocks.getLocation.mockResolvedValue({});+ mocks.insert.mockResolvedValue(undefined);+ });++ it("stores the ASN, org, and datacenter classification for a recognized IP", async () => {+ mocks.lookupAsn.mockReturnValue({ asn: 16509, organization: "AMAZON-02" });+ mocks.isDatacenterAsn.mockReturnValue(true);++ await flush();++ const row = mocks.insert.mock.calls[0][0].values[0];+ expect(row.asn).toBe(16509);+ expect(row.asn_org).toBe("AMAZON-02");+ expect(row.is_datacenter_asn).toBe(1);+ });++ it("defaults to null/empty/0 when the IP has no ASN match", async () => {+ mocks.lookupAsn.mockReturnValue(null);++ await flush();++ const row = mocks.insert.mock.calls[0][0].values[0];+ expect(row.asn).toBeNull();+ expect(row.asn_org).toBe("");+ expect(row.is_datacenter_asn).toBe(0);+ });++ it("marks residential ASNs as non-datacenter", async () => {+ mocks.lookupAsn.mockReturnValue({ asn: 7922, organization: "COMCAST-7922" });+ mocks.isDatacenterAsn.mockReturnValue(false);++ await flush();++ const row = mocks.insert.mock.calls[0][0].values[0];+ expect(row.is_datacenter_asn).toBe(0);+ });+});diff --git a/server/src/services/tracker/pageviewQueue.ts b/server/src/services/tracker/pageviewQueue.tsindex 1d9d71b12..259f4128a 100644--- a/server/src/services/tracker/pageviewQueue.ts+++ b/server/src/services/tracker/pageviewQueue.ts@@ -1,8 +1,10 @@import { DateTime } from "luxon";import { clickhouse } from "../../db/clickhouse/clickhouse.js";+import { lookupAsn } from "../../db/geolocation/asn.js";import { getLocation } from "../../db/geolocation/geolocation.js";import { createServiceLogger } from "../../lib/logger/logger.js";import { getDeviceType } from "../../utils.js";+import { isDatacenterAsn } from "./botBlocking/datacenterAsns.js";import { getChannel } from "./getChannel.js";import { clearSelfReferrer, getAllUrlParams, TotalTrackingPayload } from "./utils.js";@@ -63,6 +65,12 @@ class PageviewQueue {const city = dataForIp?.city || "";const timezone = dataForIp?.timeZone || "";+ // Same MaxMind lookup already used to decide identity bucketing+ // (bucketIpForIdentity) and bot-detection eligibility; stored here purely+ // for debugging identity-splitting reports, treated like geo data+ // (independent of storeIp) since it identifies a network, not a device.+ const asnInfo = lookupAsn(pv.ipAddress);+// Check if referrer is from the same domain and clear it if solet referrer = clearSelfReferrer(pv.referrer || "", pv.hostname || "");@@ -112,6 +120,9 @@ class PageviewQueue {tag: pv.tag || "",feature_flags: pv.feature_flags || {},import_id: null,+ asn: asnInfo?.asn ?? null,+ asn_org: asnInfo?.organization || "",+ is_datacenter_asn: asnInfo && isDatacenterAsn(asnInfo.asn) ? 1 : 0,};});