Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

ingest: floor implausible __timestamp values instead of storing 1970

Carl-Gerhard Lindesvärdaeaccdf67e17
diff --git a/packages/core/src/modules/ingest/ingest.service.test.ts b/packages/core/src/modules/ingest/ingest.service.test.ts
index 2ff3f1d29..9f0332845 100644
--- a/packages/core/src/modules/ingest/ingest.service.test.ts
+++ b/packages/core/src/modules/ingest/ingest.service.test.ts
@@ -54,6 +54,7 @@ const getProfileById = mock(async () => null);
mock.module('../../clients/geo', () => ({ getGeoLocation, getAsnInfo }));
let getOverrideDeviceId: typeof import('./ingest.service').getOverrideDeviceId;
+let getTimestamp: typeof import('./ingest.service').getTimestamp;
let handleReplay: typeof import('./ingest.service').handleReplay;
let ingestTrack: typeof import('./ingest.service').ingestTrack;
@@ -80,9 +81,8 @@ beforeAll(async () => {
...realSaltService,
getSalts,
}));
- ({ getOverrideDeviceId, handleReplay, ingestTrack } = await import(
- './ingest.service'
- ));
+ ({ getOverrideDeviceId, getTimestamp, handleReplay, ingestTrack } =
+ await import('./ingest.service'));
});
afterAll(() => {
@@ -95,6 +95,47 @@ const track = (properties?: Record<string, unknown>): ITrackHandlerPayload =>
payload: { name: 'page_view', properties },
}) as ITrackHandlerPayload;
+// Back-dating is documented and deliberate, so the guard here is a FLOOR, not
+// a window: historical imports must keep working, and only values that cannot
+// be a real event time fall back to the server clock (ISSUES.md H6).
+describe('getTimestamp', () => {
+ const ARRIVED_AT = Date.UTC(2026, 8, 27, 12, 0, 0);
+ const withTimestamp = (value: unknown) =>
+ getTimestamp(ARRIVED_AT, track({ __timestamp: value }).payload);
+
+ it('keeps an old but plausible timestamp, so imports still work', () => {
+ const result = withTimestamp('2020-01-01T00:00:00.000Z');
+ expect(result.timestamp).toBe(Date.UTC(2020, 0, 1));
+ expect(result.isTimestampFromThePast).toBe(true);
+ });
+
+ it('keeps a real epoch-millisecond value', () => {
+ const epochMs = Date.UTC(2024, 0, 15, 10, 30, 0);
+ expect(withTimestamp(epochMs).timestamp).toBe(epochMs);
+ });
+
+ it('falls back to the server clock for a value that lands in 1970', () => {
+ // `new Date(12345)` is 1970-01-01T00:00:12.345Z. One such row reached
+ // ClickHouse during the QA run and every WITH FILL series then began there.
+ expect(withTimestamp(12_345).timestamp).toBe(ARRIVED_AT);
+ expect(withTimestamp(0).timestamp).toBe(ARRIVED_AT);
+ });
+
+ it('still falls back for garbage and for the future', () => {
+ expect(withTimestamp('not a date').timestamp).toBe(ARRIVED_AT);
+ expect(withTimestamp(ARRIVED_AT + 10 * 60 * 1000).timestamp).toBe(
+ ARRIVED_AT
+ );
+ });
+
+ it('leaves a recent timestamp alone', () => {
+ const recent = ARRIVED_AT - 60 * 1000;
+ const result = withTimestamp(new Date(recent).toISOString());
+ expect(result.timestamp).toBe(recent);
+ expect(result.isTimestampFromThePast).toBe(false);
+ });
+});
+
describe('getOverrideDeviceId', () => {
it('returns a trimmed device id', () => {
expect(getOverrideDeviceId(track({ __deviceId: ' cookie-abc ' }))).toBe(
diff --git a/packages/core/src/modules/ingest/ingest.service.ts b/packages/core/src/modules/ingest/ingest.service.ts
index 410c20028..504969ea6 100644
--- a/packages/core/src/modules/ingest/ingest.service.ts
+++ b/packages/core/src/modules/ingest/ingest.service.ts
@@ -107,6 +107,15 @@ const QUEUE_PAYLOAD_HEADERS = [
const MAX_OVERRIDE_DEVICE_ID_LENGTH = 64;
const ONE_MINUTE_MS = 60 * 1000;
const FIFTEEN_MINUTES_MS = 15 * ONE_MINUTE_MS;
+/**
+ * A floor, not a window. Back-dating is a documented feature — the tracking
+ * docs describe it and the Mixpanel migration guide imports years-old events
+ * through it — so historical imports must keep working. This only catches
+ * values that cannot be a real event time: `__timestamp: 12345` is read as
+ * epoch milliseconds and lands on 1970-01-01, after which every date-filled
+ * series starts there.
+ */
+const EARLIEST_ACCEPTED_TIMESTAMP_MS = Date.UTC(2000, 0, 1);
const FALLBACK_USER_AGENT = 'unknown/1.0';
/** The whitelisted subset that ships with the queue payload. */
@@ -178,9 +187,11 @@ export function getTimestamp(
const clientTimestamp = new Date(userDefinedTimestamp);
const clientTimestampNumber = clientTimestamp.getTime();
- // Use safeTimestamp if invalid or more than 1 minute in the future
+ // Use safeTimestamp if invalid, implausibly old, or more than 1 minute
+ // in the future
if (
Number.isNaN(clientTimestampNumber) ||
+ clientTimestampNumber < EARLIEST_ACCEPTED_TIMESTAMP_MS ||
clientTimestampNumber > safeTimestamp + ONE_MINUTE_MS
) {
return { timestamp: safeTimestamp, isTimestampFromThePast: false };