Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

22.8 KB
Raw Blame
import crypto, { randomBytes } from 'crypto'
import { DateTime } from 'luxon'
import { Pool } from 'pg'
import { Readable } from 'stream'

import { Properties } from '~/plugin-scaffold'
import { ClickHouseTimestamp, ClickHouseTimestampSecondPrecision, ISOTimestamp, TimestampFormat } from '~/types'

import { logger } from './logger'
import { captureException } from './posthog'

/** Time until autoexit (due to error) gives up on graceful exit and kills the process right away. */
const GRACEFUL_EXIT_PERIOD_SECONDS = 5

export class NoRowsUpdatedError extends Error {}

export function killGracefully(): void {
    logger.error('⏲', 'Shutting nodejs services down gracefully with SIGTERM...')
    process.kill(process.pid, 'SIGTERM')
    setTimeout(() => {
        logger.error(
            '⏲',
            `Nodejs services still running after ${GRACEFUL_EXIT_PERIOD_SECONDS} s, killing it forcefully!`
        )
        process.exit(1)
    }, GRACEFUL_EXIT_PERIOD_SECONDS * 1000)
}

/**
 * @param binary Buffer
 * returns readableInstanceStream Readable
 */
export function bufferToStream(binary: Buffer): Readable {
    const readableInstanceStream = new Readable({
        read() {
            this.push(binary)
            this.push(null)
        },
    })

    return readableInstanceStream
}

export function bufferToUint32ArrayLE(buffer: Buffer): Uint32Array {
    const dataView = new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength)
    const length = buffer.byteLength / 4
    const result = new Uint32Array(length)

    for (let i = 0; i < length; i++) {
        // explicitly set little-endian
        result[i] = dataView.getUint32(i * 4, true)
    }

    return result
}

export function uint32ArrayLEToBuffer(uint32Array: Uint32Array): Buffer {
    const buffer = new ArrayBuffer(uint32Array.length * 4)
    const dataView = new DataView(buffer)

    for (let i = 0; i < uint32Array.length; i++) {
        // explicitly set little-endian
        dataView.setUint32(i * 4, uint32Array[i], true)
    }
    return Buffer.from(buffer)
}

export function createRandomUint32x4(): Uint32Array {
    const randomArray = new Uint32Array(4)
    crypto.webcrypto.getRandomValues(randomArray)
    return randomArray
}

export function cloneObject<T>(obj: T): T {
    if (obj !== Object(obj)) {
        return obj
    }
    if (Array.isArray(obj)) {
        return (obj as any[]).map(cloneObject) as unknown as T
    }
    if (obj instanceof Date) {
        return new Date(obj.getTime()) as T
    }
    if (obj instanceof DateTime) {
        return obj.toUTC() as T
    }
    const clone: Record<string, any> = {}
    for (const i in obj) {
        clone[i] = cloneObject(obj[i])
    }
    return clone as T
}

/** LUT of byte value to hexadecimal representation. For UUID stringification. */
const byteToHex: string[] = []
for (let i = 0; i < 256; i++) {
    byteToHex.push((i + 0x100).toString(16).substr(1))
}

export class UUID {
    /**
     * Check whether str
     *
     * This does not care about RFC4122, since neither does UUIDT above.
     * https://stackoverflow.com/questions/7905929/how-to-test-valid-uuid-guid
     */

    static validateString(candidate: any, throwOnInvalid = true): boolean {
        const isValid = Boolean(
            candidate &&
                typeof candidate === 'string' &&
                candidate.match(/^[0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12}$/i)
        )
        if (!isValid && throwOnInvalid) {
            throw new Error(
                'String does not match format XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX (where each X is a hexadecimal character)!'
            )
        }
        return isValid
    }

    array: Uint8Array

    constructor(candidate: string | Uint8Array | Buffer) {
        if (candidate instanceof Uint8Array) {
            if (candidate.byteLength !== 16) {
                throw new Error(`UUID must be built from exactly 16 bytes, but you provided ${candidate.byteLength}!`)
            }
            this.array = new Uint8Array(candidate)
        } else {
            candidate = candidate.trim()
            UUID.validateString(candidate)
            this.array = new Uint8Array(16)
            const characters = Array.from(candidate).filter((character) => character !== '-')
            for (let i = 0; i < characters.length; i += 2) {
                this.array[i / 2] = parseInt(characters[i] + characters[i + 1], 16)
            }
        }
    }

    /** Convert to 128-bit BigInt. */
    valueOf(): bigint {
        let value = 0n
        for (const byte of this.array) {
            value <<= 8n
            value += BigInt(byte)
        }
        return value
    }

    /**
     * Convert to string format of the form:
     * XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
     */
    toString(): string {
        // Adapted from https://github.com/uuidjs/uuid/blob/master/src/stringify.js
        // Note: Be careful editing this code! It's been tuned for performance and works in ways you may not expect.
        // See https://github.com/uuidjs/uuid/pull/434
        const arr = this.array
        return (
            byteToHex[arr[0]] +
            byteToHex[arr[1]] +
            byteToHex[arr[2]] +
            byteToHex[arr[3]] +
            '-' +
            byteToHex[arr[4]] +
            byteToHex[arr[5]] +
            '-' +
            byteToHex[arr[6]] +
            byteToHex[arr[7]] +
            '-' +
            byteToHex[arr[8]] +
            byteToHex[arr[9]] +
            '-' +
            byteToHex[arr[10]] +
            byteToHex[arr[11]] +
            byteToHex[arr[12]] +
            byteToHex[arr[13]] +
            byteToHex[arr[14]] +
            byteToHex[arr[15]]
        ).toLowerCase()
    }
}

// Lowercase UUID session IDs for consistency. Some mobile apps send uppercase
// session IDs, which causes issues on query paths where the session ID is parsed
// as a UUID and converted back to a string.
// See https://github.com/PostHog/posthog/issues/46111
export function normalizeSessionId(sessionId: string): string {
    return UUID.validateString(sessionId, false) ? sessionId.toLowerCase() : sessionId
}

/**
 * UUID (mostly) sortable by generation time.
 *
 * Note: this class was created in 2020, before UUIDv7 was published in 2024 https://www.rfc-editor.org/rfc/rfc9562.
 * Nowadays, you probably want to use UUIDv7 instead, as many DB engines have first-class support for them
 * (e.g. clickhouse has [UUIDv7ToDateTime](https://clickhouse.com/docs/sql-reference/functions/uuid-functions#UUIDv7ToDateTime))
 *
 * Original comment from 2020 continues:
 *
 * This doesn't adhere to any official UUID version spec, but it is superior as a primary key:
 * to incremented integers (as they can reveal sensitive business information about usage volumes and patterns),
 * to UUID v4 (as the complete randomness of v4 makes its indexing performance suboptimal),
 * and to UUID v1 (as despite being time-based it can't be used practically for sorting by generation time).
 *
 * Order can be messed up if system Unix time is changed or if more than 65 536 IDs are generated per millisecond
 * (that's over 5 trillion events per day), but it should be largely safe to assume that these are time-sortable.
 *
 * Anatomy:
 * - 6 bytes - Unix time milliseconds unsigned integer
 * - 2 bytes - autoincremented series unsigned integer (per millisecond, rolls over to 0 after reaching 65 535 UUIDs in one ms)
 * - 8 bytes - securely random gibberish
 *
 * Loosely based on [Segment's KSUID](https://github.com/segmentio/ksuid) and
 * on [Twitter's snowflake ID](https://blog.twitter.com/engineering/en_us/a/2010/announcing-snowflake.html).
 * Ported from the PostHog Django app.
 */
export class UUIDT extends UUID {
    static currentSeriesPerMs: Map<number, number> = new Map()

    /** Get per-millisecond series integer in range [0-65536). */
    static getSeries(unixTimeMs: number): number {
        const series = UUIDT.currentSeriesPerMs.get(unixTimeMs) ?? 0
        if (UUIDT.currentSeriesPerMs.size > 10_000) {
            // Clear class dict periodically
            UUIDT.currentSeriesPerMs.clear()
        }
        UUIDT.currentSeriesPerMs.set(unixTimeMs, (series + 1) % 65_536)
        return series
    }

    constructor(unixTimeMs?: number) {
        if (!unixTimeMs) {
            unixTimeMs = DateTime.utc().toMillis()
        }
        let unixTimeMsBig = BigInt(unixTimeMs)
        let series = UUIDT.getSeries(unixTimeMs)
        // 64 bits (8 bytes) total
        const array = new Uint8Array(16)
        // 48 bits for time, WILL FAIL in 10 895 CE
        // XXXXXXXX-XXXX-****-****-************
        for (let i = 5; i >= 0; i--) {
            array[i] = Number(unixTimeMsBig & 0xffn) // use last 8 binary digits to set UUID 2 hexadecimal digits
            unixTimeMsBig >>= 8n // remove these last 8 binary digits
        }
        // 16 bits for series
        // ********-****-XXXX-****-************
        for (let i = 7; i >= 6; i--) {
            array[i] = series & 0xff // use last 8 binary digits to set UUID 2 hexadecimal digits
            series >>>= 8 // remove these last 8 binary digits
        }
        // 64 bits for random gibberish
        // ********-****-****-XXXX-XXXXXXXXXXXX
        array.set(randomBytes(8), 8)
        super(array)
    }
}

export class UUID7 extends UUID {
    constructor(bufferOrUnixTimeMs?: number | Buffer, rand?: Buffer) {
        if (bufferOrUnixTimeMs instanceof Buffer) {
            if (bufferOrUnixTimeMs.length !== 16) {
                throw new Error(`UUID7 from buffer requires 16 bytes, got ${bufferOrUnixTimeMs.length}`)
            }
            super(bufferOrUnixTimeMs)
            return
        }
        const unixTimeMs = (bufferOrUnixTimeMs as number | undefined) ?? DateTime.utc().toMillis()
        let unixTimeMsBig = BigInt(unixTimeMs)

        if (!rand) {
            rand = randomBytes(10)
        } else if (rand.length !== 10) {
            throw new Error(`UUID7 requires 10 bytes of random data, got ${rand.length}`)
        }

        // see https://www.rfc-editor.org/rfc/rfc9562#name-uuid-version-7
        // a UUIDv7 is 128 bits (16 bytes) total
        // 48 bits for unix_ts_ms,
        // 4 bits for ver = 0b111 (7)
        // 12 bits for rand_a
        // 2 bits for var = 0b10
        // 62 bits for rand_b
        // we set fully random values for rand_a and rand_b

        const array = new Uint8Array(16)
        // 48 bits for time, WILL FAIL in 10 895 CE
        // XXXXXXXX-XXXX-****-****-************
        for (let i = 5; i >= 0; i--) {
            array[i] = Number(unixTimeMsBig & 0xffn) // use last 8 binary digits to set UUID 2 hexadecimal digits
            unixTimeMsBig >>= 8n // remove these last 8 binary digits
        }
        // rand_a and rand_b
        // ********-****-*XXX-XXXX-XXXXXXXXXXXX
        array.set(rand, 6)

        // ver and var
        // ********-****-7***-X***-************
        array[6] = 0b0111_0000 | (array[6] & 0b0000_1111)
        array[8] = 0b1000_0000 | (array[8] & 0b0011_1111)

        super(array)
    }
}

/* Format timestamps.
Allowed timestamp formats support ISO and ClickHouse formats according to
`timestampFormat`. This distinction is relevant because ClickHouse does NOT
 necessarily accept all possible ISO timestamps. */
export function castTimestampOrNow(
    timestamp?: DateTime | string | null,
    timestampFormat?: TimestampFormat.ISO
): ISOTimestamp
export function castTimestampOrNow(
    timestamp: DateTime | string | null,
    timestampFormat: TimestampFormat.ClickHouse
): ClickHouseTimestamp
export function castTimestampOrNow(
    timestamp: DateTime | string | null,
    timestampFormat: TimestampFormat.ClickHouseSecondPrecision
): ClickHouseTimestampSecondPrecision
export function castTimestampOrNow(
    timestamp?: DateTime | string | null,
    timestampFormat: TimestampFormat = TimestampFormat.ISO
): ISOTimestamp | ClickHouseTimestamp | ClickHouseTimestampSecondPrecision {
    if (!timestamp) {
        timestamp = DateTime.utc()
    } else if (typeof timestamp === 'string') {
        timestamp = DateTime.fromISO(timestamp)
    }

    return castTimestampToClickhouseFormat(timestamp, timestampFormat)
}

const DATETIME_FORMAT_CLICKHOUSE_SECOND_PRECISION = 'yyyy-MM-dd HH:mm:ss'
const DATETIME_FORMAT_CLICKHOUSE = 'yyyy-MM-dd HH:mm:ss.u'

export function castTimestampToClickhouseFormat(timestamp: DateTime, timestampFormat: TimestampFormat.ISO): ISOTimestamp
export function castTimestampToClickhouseFormat(
    timestamp: DateTime,
    timestampFormat: TimestampFormat.ClickHouse
): ClickHouseTimestamp
export function castTimestampToClickhouseFormat(
    timestamp: DateTime,
    timestampFormat: TimestampFormat.ClickHouseSecondPrecision
): ClickHouseTimestampSecondPrecision
export function castTimestampToClickhouseFormat(
    timestamp: DateTime,
    timestampFormat: TimestampFormat
): ISOTimestamp | ClickHouseTimestamp | ClickHouseTimestampSecondPrecision
export function castTimestampToClickhouseFormat(
    timestamp: DateTime,
    timestampFormat: TimestampFormat = TimestampFormat.ISO
): ISOTimestamp | ClickHouseTimestamp | ClickHouseTimestampSecondPrecision {
    timestamp = timestamp.toUTC()
    switch (timestampFormat) {
        case TimestampFormat.ClickHouseSecondPrecision:
            return timestamp.toFormat(DATETIME_FORMAT_CLICKHOUSE_SECOND_PRECISION) as ClickHouseTimestampSecondPrecision
        case TimestampFormat.ClickHouse:
            return timestamp.toFormat(DATETIME_FORMAT_CLICKHOUSE) as ClickHouseTimestamp
        case TimestampFormat.ISO:
            return timestamp.toUTC().toISO() as ISOTimestamp
        default:
            throw new Error(`Unrecognized timestamp format ${timestampFormat}!`)
    }
}

// Used only when parsing clickhouse timestamps
export function clickHouseTimestampToDateTime(timestamp: ClickHouseTimestamp): DateTime {
    return DateTime.fromFormat(timestamp, DATETIME_FORMAT_CLICKHOUSE, { zone: 'UTC' })
}

export function clickHouseTimestampToISO(timestamp: ClickHouseTimestamp): ISOTimestamp {
    return clickHouseTimestampToDateTime(timestamp).toISO() as ISOTimestamp
}

export function clickHouseTimestampSecondPrecisionToISO(timestamp: ClickHouseTimestamp): ISOTimestamp {
    return DateTime.fromFormat(timestamp, DATETIME_FORMAT_CLICKHOUSE_SECOND_PRECISION, {
        zone: 'UTC',
    }).toISO() as ISOTimestamp
}

export function delay(ms: number): Promise<void> {
    return new Promise((resolve) => {
        setTimeout(resolve, ms)
    })
}

/** Remove all quotes from the provided identifier to prevent SQL injection. */
export function sanitizeSqlIdentifier(unquotedIdentifier: string): string {
    return unquotedIdentifier.replace(/[^\w\d_]+/g, '')
}

/** Escape single quotes and slashes */
export function escapeClickHouseString(string: string): string {
    // In string literals, you need to escape at least `'` and `\`.
    // https://clickhouse.tech/docs/en/sql-reference/syntax/
    return string.replace(/\\/g, '\\\\').replace(/'/g, "\\'")
}

/** Standardize JS code used internally to form without extraneous indentation. Template literal function. */
export function code(strings: TemplateStringsArray): string {
    const stringsConcat = strings.join('…')
    const indentation = stringsConcat.match(/^\n([ ]*)/)?.[1].length ?? 0
    const dedentedCode = stringsConcat.replace(new RegExp(`^[ ]{${indentation}}`, 'gm'), '')
    return dedentedCode.trim()
}

export async function tryTwice<T>(callback: () => Promise<T>, errorMessage: string, timeoutMs = 5000): Promise<T> {
    const timeout = new Promise((_, reject) => setTimeout(reject, timeoutMs))
    try {
        const response = await Promise.race([timeout, callback()])
        return response as T
    } catch {
        captureException(`Had to run twice: ${errorMessage}`)
        // try one more time
        return await callback()
    }
}

export function createPostgresPool(
    connectionString: string,
    poolSize: number,
    applicationName: string,
    onError?: (error: Error) => any,
    connectionTimeoutMillis?: number
): Pool {
    const pgPool = new Pool({
        connectionString,
        idleTimeoutMillis: 500,
        application_name: applicationName,
        max: poolSize,
        connectionTimeoutMillis,
    })

    const handleError =
        onError ||
        ((error) => {
            captureException(error)
            logger.error('🔴', 'PostgreSQL error encountered!\n', error)
        })

    pgPool.on('error', handleError)

    return pgPool
}

export function groupBy<T extends Record<string, any>, K extends keyof T>(
    objects: T[],
    key: K,
    flat?: false
): Record<T[K], T[]>
export function groupBy<T extends Record<string, any>, K extends keyof T>(
    objects: T[],
    key: K,
    flat: true
): Record<T[K], T>
export function groupBy<T extends Record<string, any>, K extends keyof T>(
    objects: T[],
    key: K,
    flat = false
): Record<T[K], T[] | T> {
    return flat
        ? objects.reduce(
              (grouping, currentItem) => {
                  if (currentItem[key] in grouping) {
                      throw new Error(
                          `Key "${String(key)}" has more than one matching value, which is not allowed in flat groupBy!`
                      )
                  }
                  grouping[currentItem[key]] = currentItem
                  return grouping
              },
              {} as Record<T[K], T>
          )
        : objects.reduce(
              (grouping, currentItem) => {
                  ;(grouping[currentItem[key]] = grouping[currentItem[key]] || []).push(currentItem)
                  return grouping
              },
              {} as Record<T[K], T[]>
          )
}

export function clamp(value: number, min: number, max: number): number {
    return value > max ? max : value < min ? min : value
}

export function stringClamp(value: string, def: number, min: number, max: number): number {
    const nanToNull = (nr: number): null | number => (isNaN(nr) ? null : nr)
    return clamp(nanToNull(parseInt(value)) ?? def, min, max)
}

export function stringify(value: any): string {
    switch (typeof value) {
        case 'string':
            return value
        case 'undefined':
            return 'undefined'
        default:
            return JSON.stringify(value)
    }
}

export class IllegalOperationError extends Error {
    override name = 'IllegalOperationError'

    constructor(operation: string) {
        super(operation)
    }
}

export function getByAge<K, V>(cache: Map<K, [V, number]>, key: K, maxAgeMs = 30_000): V | undefined {
    if (cache.has(key)) {
        const [value, age] = cache.get(key)!
        if (Date.now() - age <= maxAgeMs) {
            return value
        }
    }
    return undefined
}

// Equivalent of Python's string.ascii_letters
export function getAsciiLetters(): string {
    const LOWERCASE_START_POINT = 97 // ASCII 'a'
    const UPPERCASE_START_POINT = 65 // ASCII 'A'

    const lowercaseLetters = Array.from({ length: 26 }).map((_, i) => String.fromCharCode(LOWERCASE_START_POINT + i))
    const uppercaseLetters = Array.from({ length: 26 }).map((_, i) => String.fromCharCode(UPPERCASE_START_POINT + i))

    return `${lowercaseLetters.join('')}${uppercaseLetters.join('')}`
}

// Equivalent of Python's string.digits
export function getAllDigits(): string {
    return Array.from({ length: 10 })
        .map((_, i) => i)
        .join('')
}

export function generateRandomToken(nBytes: number): string {
    return intToBase(Number.parseInt(randomBytes(nBytes).toString('hex'), 16), 62)
}

export function intToBase(num: number, base: number): string {
    if (base > 62) {
        throw new IllegalOperationError('Cannot convert integer to base above 62')
    }
    const alphabet = getAllDigits() + getAsciiLetters()
    if (num < 0) {
        return '-' + intToBase(-num, base)
    }
    let value = ''
    while (num != 0) {
        const oldNum = num
        num = Math.floor(oldNum / alphabet.length)
        const index = oldNum % alphabet.length
        value = alphabet[index] + value
    }

    return value || '0'
}

// For errors we want to explicitly throw
// concerning race conditions across threads
export class RaceConditionError extends Error {
    override name = 'RaceConditionError'
}

/** Get a value from a properties object by its path. This allows accessing nested properties. */
export function getPropertyValueByPath(properties: Properties, [firstKey, ...nestedKeys]: string[]): any {
    if (firstKey === undefined) {
        throw new Error('No path to property was provided')
    }
    let value = properties[firstKey]
    for (const key of nestedKeys) {
        if (value === undefined) {
            return undefined
        }
        value = value[key]
    }
    return value
}

export async function sleep(ms: number): Promise<void> {
    return new Promise((resolve) => setTimeout(resolve, ms))
}

// Values of the $lib property that have been seen in the wild
export const KNOWN_LIB_VALUES = new Set([
    'web',
    'posthog-python',
    '',
    'js',
    'posthog-node',
    'posthog-react-native',
    'posthog-ruby',
    'posthog-ios',
    'posthog-android',
    'Segment',
    'posthog-go',
    'analytics-node',
    'RudderLabs JavaScript SDK',
    'mobile',
    'posthog-php',
    'posthog-laravel',
    'zapier',
    'Webflow',
    'posthog-flutter',
    'com.rudderstack.android.sdk.core',
    'rudder-analytics-python',
    'rudder-ios-library',
    'rudder-analytics-php',
    'macos',
    'service_data',
    'flow',
    'PROD',
    'unknown',
    'api',
    'unbounce',
    'backend',
    'analytics-python',
    'windows',
    'cf-analytics-go',
    'server',
    'core',
    'Marketing',
    'Product',
    'com.rudderstack.android.sdk',
    'net-gibraltar',
    'posthog-java',
    'rudderanalytics-ruby',
    'GSHEETS_AIRBYTE',
    'posthog-plugin-server',
    'DotPostHog',
    'analytics-go',
    'serverless',
    'wordpress',
    'hog_function',
    'http',
    'desktop',
    'elixir',
    'DEV',
    'RudderAnalytics.NET',
    'PR',
    'railway',
    'HTTP',
    'extension',
    'cyclotron-testing',
    'RudderStack Shopify Cloud',
    'GSHEETS_MONITOR',
    'Rudder',
    'API',
    'rudder-sdk-ruby-sync',
    'curl',
])

export const getKnownLibValueOrSentinel = (lib: string): string => {
    if (lib === '') {
        return '$empty'
    }
    if (!lib) {
        return '$nil'
    }
    if (KNOWN_LIB_VALUES.has(lib)) {
        return lib
    }
    return '$other'
}

// Check if 2 maps with primitive values are equal
export const areMapsEqual = <K, V>(map1: Map<K, V>, map2: Map<K, V>): boolean => {
    if (map1.size !== map2.size) {
        return false
    }
    for (const [key, value] of map1) {
        if (!map2.has(key) || map2.get(key) !== value) {
            return false
        }
    }
    return true
}

export function promisifyCallback<TResult>(fn: (cb: (err: any, result?: TResult) => void) => void): Promise<TResult> {
    return new Promise<TResult>((resolve, reject) => {
        fn((err, result) => {
            if (err) {
                reject(err)
            } else {
                resolve(result as TResult)
            }
        })
    })
}