Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

3.2 KB
Raw Blame
import { DateTime } from 'luxon'

import { PipelineWarning } from '~/ingestion/framework/pipeline.interface'
import { PipelineResult, drop, ok } from '~/ingestion/framework/results'
import { ProcessingStep } from '~/ingestion/framework/steps'
import { EventHeaders, PipelineEvent, Team } from '~/types'

export interface DropOldEventsInput {
    event: PipelineEvent
    team: Team
    headers: Pick<EventHeaders, 'timestamp' | 'now'>
}

/**
 * Creates a pipeline step that drops events older than a team's configured threshold.
 *
 * If an event is too old, it returns a `drop` result with an ingestion warning
 * that will be sent to Kafka by `handleIngestionWarnings`.
 *
 * Uses the `timestamp` and `now` headers set by the Rust capture service.
 * The timestamp header contains milliseconds since epoch (already normalized with clock skew correction).
 * The now header contains the time when the event was received by the capture service.
 */
export function createDropOldEventsStep<T extends DropOldEventsInput>(): ProcessingStep<T, T> {
    return function dropOldEventsStep(input: T): Promise<PipelineResult<T>> {
        const { event, team, headers } = input

        // If no drop threshold is set (null) or set to 0, don't drop any events
        // Zero threshold is ignored to protect from misconfiguration bugs
        if (!team.drop_events_older_than_seconds) {
            return Promise.resolve(ok(input))
        }

        const eventTimestamp = parseEventTimestampFromHeaders(headers)
        if (!eventTimestamp) {
            return Promise.resolve(ok(input))
        }

        const now = parseNowFromHeaders(headers)
        const ageInSeconds = calculateAgeInSeconds(eventTimestamp, now)

        if (ageInSeconds > team.drop_events_older_than_seconds) {
            const warning: PipelineWarning = {
                type: 'event_dropped_too_old',
                details: {
                    eventUuid: event.uuid,
                    event: event.event,
                    distinctId: event.distinct_id,
                    eventTimestamp: eventTimestamp.toISO(),
                    ageInSeconds: Math.floor(ageInSeconds),
                    dropThresholdSeconds: team.drop_events_older_than_seconds,
                },
                alwaysSend: false,
            }
            return Promise.resolve(drop('event_too_old', [], [warning]))
        }

        return Promise.resolve(ok(input))
    }
}

function parseEventTimestampFromHeaders(headers: Pick<EventHeaders, 'timestamp'> | undefined): DateTime | undefined {
    if (!headers?.timestamp) {
        return undefined
    }

    const timestampMs = parseInt(headers.timestamp, 10)
    if (!Number.isFinite(timestampMs)) {
        return undefined
    }

    const eventTimestamp = DateTime.fromMillis(timestampMs)
    return eventTimestamp.isValid ? eventTimestamp : undefined
}

function parseNowFromHeaders(headers: Pick<EventHeaders, 'now'> | undefined): DateTime {
    if (headers?.now instanceof Date && !isNaN(headers.now.getTime())) {
        const now = DateTime.fromJSDate(headers.now)
        if (now.isValid) {
            return now
        }
    }
    return DateTime.utc()
}

function calculateAgeInSeconds(eventTimestamp: DateTime, now: DateTime): number {
    return now.diff(eventTimestamp, 'seconds').seconds
}