OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
ingest: floor implausible __timestamp values instead of storing 1970
Carl-Gerhard Lindesvärdaeaccdf67e17
diff --git a/packages/core/src/modules/ingest/ingest.service.test.ts b/packages/core/src/modules/ingest/ingest.service.test.tsindex 2ff3f1d29..9f0332845 100644--- a/packages/core/src/modules/ingest/ingest.service.test.ts+++ b/packages/core/src/modules/ingest/ingest.service.test.ts@@ -54,6 +54,7 @@ const getProfileById = mock(async () => null);mock.module('../../clients/geo', () => ({ getGeoLocation, getAsnInfo }));let getOverrideDeviceId: typeof import('./ingest.service').getOverrideDeviceId;+let getTimestamp: typeof import('./ingest.service').getTimestamp;let handleReplay: typeof import('./ingest.service').handleReplay;let ingestTrack: typeof import('./ingest.service').ingestTrack;@@ -80,9 +81,8 @@ beforeAll(async () => {...realSaltService,getSalts,}));- ({ getOverrideDeviceId, handleReplay, ingestTrack } = await import(- './ingest.service'- ));+ ({ getOverrideDeviceId, getTimestamp, handleReplay, ingestTrack } =+ await import('./ingest.service'));});afterAll(() => {@@ -95,6 +95,47 @@ const track = (properties?: Record<string, unknown>): ITrackHandlerPayload =>payload: { name: 'page_view', properties },}) as ITrackHandlerPayload;+// Back-dating is documented and deliberate, so the guard here is a FLOOR, not+// a window: historical imports must keep working, and only values that cannot+// be a real event time fall back to the server clock (ISSUES.md H6).+describe('getTimestamp', () => {+ const ARRIVED_AT = Date.UTC(2026, 8, 27, 12, 0, 0);+ const withTimestamp = (value: unknown) =>+ getTimestamp(ARRIVED_AT, track({ __timestamp: value }).payload);++ it('keeps an old but plausible timestamp, so imports still work', () => {+ const result = withTimestamp('2020-01-01T00:00:00.000Z');+ expect(result.timestamp).toBe(Date.UTC(2020, 0, 1));+ expect(result.isTimestampFromThePast).toBe(true);+ });++ it('keeps a real epoch-millisecond value', () => {+ const epochMs = Date.UTC(2024, 0, 15, 10, 30, 0);+ expect(withTimestamp(epochMs).timestamp).toBe(epochMs);+ });++ it('falls back to the server clock for a value that lands in 1970', () => {+ // `new Date(12345)` is 1970-01-01T00:00:12.345Z. One such row reached+ // ClickHouse during the QA run and every WITH FILL series then began there.+ expect(withTimestamp(12_345).timestamp).toBe(ARRIVED_AT);+ expect(withTimestamp(0).timestamp).toBe(ARRIVED_AT);+ });++ it('still falls back for garbage and for the future', () => {+ expect(withTimestamp('not a date').timestamp).toBe(ARRIVED_AT);+ expect(withTimestamp(ARRIVED_AT + 10 * 60 * 1000).timestamp).toBe(+ ARRIVED_AT+ );+ });++ it('leaves a recent timestamp alone', () => {+ const recent = ARRIVED_AT - 60 * 1000;+ const result = withTimestamp(new Date(recent).toISOString());+ expect(result.timestamp).toBe(recent);+ expect(result.isTimestampFromThePast).toBe(false);+ });+});+describe('getOverrideDeviceId', () => {it('returns a trimmed device id', () => {expect(getOverrideDeviceId(track({ __deviceId: ' cookie-abc ' }))).toBe(diff --git a/packages/core/src/modules/ingest/ingest.service.ts b/packages/core/src/modules/ingest/ingest.service.tsindex 410c20028..504969ea6 100644--- a/packages/core/src/modules/ingest/ingest.service.ts+++ b/packages/core/src/modules/ingest/ingest.service.ts@@ -107,6 +107,15 @@ const QUEUE_PAYLOAD_HEADERS = [const MAX_OVERRIDE_DEVICE_ID_LENGTH = 64;const ONE_MINUTE_MS = 60 * 1000;const FIFTEEN_MINUTES_MS = 15 * ONE_MINUTE_MS;+/**+ * A floor, not a window. Back-dating is a documented feature — the tracking+ * docs describe it and the Mixpanel migration guide imports years-old events+ * through it — so historical imports must keep working. This only catches+ * values that cannot be a real event time: `__timestamp: 12345` is read as+ * epoch milliseconds and lands on 1970-01-01, after which every date-filled+ * series starts there.+ */+const EARLIEST_ACCEPTED_TIMESTAMP_MS = Date.UTC(2000, 0, 1);const FALLBACK_USER_AGENT = 'unknown/1.0';/** The whitelisted subset that ships with the queue payload. */@@ -178,9 +187,11 @@ export function getTimestamp(const clientTimestamp = new Date(userDefinedTimestamp);const clientTimestampNumber = clientTimestamp.getTime();- // Use safeTimestamp if invalid or more than 1 minute in the future+ // Use safeTimestamp if invalid, implausibly old, or more than 1 minute+ // in the futureif (Number.isNaN(clientTimestampNumber) ||+ clientTimestampNumber < EARLIEST_ACCEPTED_TIMESTAMP_MS ||clientTimestampNumber > safeTimestamp + ONE_MINUTE_MS) {return { timestamp: safeTimestamp, isTimestampFromThePast: false };