Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

fix(analytics): update processResults to preserve whitespace-only strings and prevent coercion of non-finite numeric strings

Bill Yang613cd0153c77
diff --git a/server/src/api/analytics/utils/utils.test.ts b/server/src/api/analytics/utils/utils.test.ts
index 91b33bfe8..78c92eb41 100644
--- a/server/src/api/analytics/utils/utils.test.ts
+++ b/server/src/api/analytics/utils/utils.test.ts
@@ -104,12 +104,37 @@ describe("processResults", () => {
expect(rows).toEqual([{ browser: "Chrome", date: "2024-01-01", version: "1.2.3", word: "true" }]);
});
- // Number(" ") === 0, and only the exact empty string is excluded, so
- // whitespace-only strings collapse to 0. Pinning current behavior.
- it("should coerce whitespace-only strings to 0", async () => {
- const rows = await processResults<Record<string, unknown>>(makeResultSet([{ blank: " " }]));
+ it("should leave whitespace-only strings untouched", async () => {
+ const rows = await processResults<Record<string, unknown>>(makeResultSet([{ blank: " ", tab: "\t" }]));
- expect(rows).toEqual([{ blank: 0 }]);
+ expect(rows).toEqual([{ blank: " ", tab: "\t" }]);
+ });
+
+ // An 18-digit Meta campaign ID exceeds Number.MAX_SAFE_INTEGER: coercing it
+ // rounded 120248430174340693 to 120248430174340690 and corrupted the value
+ // shown for utm_campaign, utm_content and every other text column.
+ it("should not coerce numeric strings that a double cannot represent exactly", async () => {
+ const rows = await processResults<Record<string, unknown>>(
+ makeResultSet([{ value: "120248430174340693", count: "42" }])
+ );
+
+ expect(rows).toEqual([{ value: "120248430174340693", count: 42 }]);
+ });
+
+ it("should not coerce numeric strings whose text would not round-trip", async () => {
+ const rows = await processResults<Record<string, unknown>>(
+ makeResultSet([{ leadingZeros: "007", plus: "+5", trailingZero: "1.50", exponent: "1e5" }])
+ );
+
+ expect(rows).toEqual([{ leadingZeros: "007", plus: "+5", trailingZero: "1.50", exponent: "1e5" }]);
+ });
+
+ it("should not coerce strings named after non-finite numbers", async () => {
+ const rows = await processResults<Record<string, unknown>>(
+ makeResultSet([{ a: "NaN", b: "Infinity", c: "-Infinity" }])
+ );
+
+ expect(rows).toEqual([{ a: "NaN", b: "Infinity", c: "-Infinity" }]);
});
it("should process every row independently", async () => {
diff --git a/server/src/api/analytics/utils/utils.ts b/server/src/api/analytics/utils/utils.ts
index 3c69c54a1..3512ef491 100644
--- a/server/src/api/analytics/utils/utils.ts
+++ b/server/src/api/analytics/utils/utils.ts
@@ -3,26 +3,37 @@ import { and, eq, inArray } from "drizzle-orm";
import { db } from "../../../db/postgres/postgres.js";
import { userProfiles } from "../../../db/postgres/schema.js";
+/**
+ * ClickHouse serialises 64-bit integers (and Decimals) as JSON strings, so
+ * numeric-looking string columns are coerced back to numbers here.
+ *
+ * The coercion is lossless-only: a string becomes a number solely when
+ * `String(Number(v))` reproduces the original text exactly. Anything a double
+ * cannot represent — an 18-digit ad-platform campaign ID like
+ * `120248430174340693`, an identifier with leading zeros — stays text instead of
+ * being silently rounded to the nearest representable value. Counts and
+ * percentages round-trip cleanly and are unaffected.
+ */
+function coerceNumericString(value: unknown): unknown {
+ if (typeof value !== "string") return value;
+
+ const asNumber = Number(value);
+ // Rejects NaN and Infinity, which would otherwise round-trip through String()
+ // and turn values literally named "NaN" or "Infinity" into numbers.
+ if (!Number.isFinite(asNumber)) return value;
+
+ return String(asNumber) === value ? asNumber : value;
+}
+
export async function processResults<T>(results: ResultSet<"JSONEachRow">): Promise<T[]> {
const data: T[] = await results.json();
for (const row of data) {
for (const key in row) {
- // Only convert to number if the value is not null/undefined and is a valid number
- if (
- key !== "session_id" &&
- key !== "user_id" &&
- key !== "identified_user_id" &&
- key !== "effective_user_id" &&
- row[key] !== null &&
- row[key] !== undefined &&
- row[key] !== "" &&
- row[key] !== true &&
- row[key] !== false &&
- !Array.isArray(row[key]) &&
- !isNaN(Number(row[key]))
- ) {
- row[key] = Number(row[key]) as any;
+ // Identifiers are opaque text even when they look numeric.
+ if (key === "session_id" || key === "user_id" || key === "identified_user_id" || key === "effective_user_id") {
+ continue;
}
+ row[key] = coerceNumericString(row[key]) as any;
}
}
return data;