🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
fix(analytics): update processResults to preserve whitespace-only strings and prevent coercion of non-finite numeric strings
Bill Yang613cd0153c77
diff --git a/server/src/api/analytics/utils/utils.test.ts b/server/src/api/analytics/utils/utils.test.tsindex 91b33bfe8..78c92eb41 100644--- a/server/src/api/analytics/utils/utils.test.ts+++ b/server/src/api/analytics/utils/utils.test.ts@@ -104,12 +104,37 @@ describe("processResults", () => {expect(rows).toEqual([{ browser: "Chrome", date: "2024-01-01", version: "1.2.3", word: "true" }]);});- // Number(" ") === 0, and only the exact empty string is excluded, so- // whitespace-only strings collapse to 0. Pinning current behavior.- it("should coerce whitespace-only strings to 0", async () => {- const rows = await processResults<Record<string, unknown>>(makeResultSet([{ blank: " " }]));+ it("should leave whitespace-only strings untouched", async () => {+ const rows = await processResults<Record<string, unknown>>(makeResultSet([{ blank: " ", tab: "\t" }]));- expect(rows).toEqual([{ blank: 0 }]);+ expect(rows).toEqual([{ blank: " ", tab: "\t" }]);+ });++ // An 18-digit Meta campaign ID exceeds Number.MAX_SAFE_INTEGER: coercing it+ // rounded 120248430174340693 to 120248430174340690 and corrupted the value+ // shown for utm_campaign, utm_content and every other text column.+ it("should not coerce numeric strings that a double cannot represent exactly", async () => {+ const rows = await processResults<Record<string, unknown>>(+ makeResultSet([{ value: "120248430174340693", count: "42" }])+ );++ expect(rows).toEqual([{ value: "120248430174340693", count: 42 }]);+ });++ it("should not coerce numeric strings whose text would not round-trip", async () => {+ const rows = await processResults<Record<string, unknown>>(+ makeResultSet([{ leadingZeros: "007", plus: "+5", trailingZero: "1.50", exponent: "1e5" }])+ );++ expect(rows).toEqual([{ leadingZeros: "007", plus: "+5", trailingZero: "1.50", exponent: "1e5" }]);+ });++ it("should not coerce strings named after non-finite numbers", async () => {+ const rows = await processResults<Record<string, unknown>>(+ makeResultSet([{ a: "NaN", b: "Infinity", c: "-Infinity" }])+ );++ expect(rows).toEqual([{ a: "NaN", b: "Infinity", c: "-Infinity" }]);});it("should process every row independently", async () => {diff --git a/server/src/api/analytics/utils/utils.ts b/server/src/api/analytics/utils/utils.tsindex 3c69c54a1..3512ef491 100644--- a/server/src/api/analytics/utils/utils.ts+++ b/server/src/api/analytics/utils/utils.ts@@ -3,26 +3,37 @@ import { and, eq, inArray } from "drizzle-orm";import { db } from "../../../db/postgres/postgres.js";import { userProfiles } from "../../../db/postgres/schema.js";+/**+ * ClickHouse serialises 64-bit integers (and Decimals) as JSON strings, so+ * numeric-looking string columns are coerced back to numbers here.+ *+ * The coercion is lossless-only: a string becomes a number solely when+ * `String(Number(v))` reproduces the original text exactly. Anything a double+ * cannot represent — an 18-digit ad-platform campaign ID like+ * `120248430174340693`, an identifier with leading zeros — stays text instead of+ * being silently rounded to the nearest representable value. Counts and+ * percentages round-trip cleanly and are unaffected.+ */+function coerceNumericString(value: unknown): unknown {+ if (typeof value !== "string") return value;++ const asNumber = Number(value);+ // Rejects NaN and Infinity, which would otherwise round-trip through String()+ // and turn values literally named "NaN" or "Infinity" into numbers.+ if (!Number.isFinite(asNumber)) return value;++ return String(asNumber) === value ? asNumber : value;+}+export async function processResults<T>(results: ResultSet<"JSONEachRow">): Promise<T[]> {const data: T[] = await results.json();for (const row of data) {for (const key in row) {- // Only convert to number if the value is not null/undefined and is a valid number- if (- key !== "session_id" &&- key !== "user_id" &&- key !== "identified_user_id" &&- key !== "effective_user_id" &&- row[key] !== null &&- row[key] !== undefined &&- row[key] !== "" &&- row[key] !== true &&- row[key] !== false &&- !Array.isArray(row[key]) &&- !isNaN(Number(row[key]))- ) {- row[key] = Number(row[key]) as any;+ // Identifiers are opaque text even when they look numeric.+ if (key === "session_id" || key === "user_id" || key === "identified_user_id" || key === "effective_user_id") {+ continue;}+ row[key] = coerceNumericString(row[key]) as any;}}return data;