Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

9.8 KB
Raw Blame
import { createHash } from 'crypto'

import { RedisClientPipeline, RedisV2 } from '~/common/redis/redis-v2'

import {
    CyclotronJobInvocation,
    CyclotronJobInvocationHogFlow,
    CyclotronJobInvocationHogFunction,
    HogFunctionMasking,
} from '../../types'
import { dualRead } from '../../utils/dual-store'
import { execHog } from '../../utils/hog-exec'

export const BASE_REDIS_KEY = process.env.NODE_ENV == 'test' ? '@posthog-test/hog-masker' : '@posthog/hog-masker'
const REDIS_KEY_TOKENS = `${BASE_REDIS_KEY}/mask`

// NOTE: These are controlled via the api so are more of a sanity fallback
const MASKER_MAX_TTL_HOG_FUNCTION = 60 * 60 * 24
const MASKER_MAX_TTL_HOG_FLOW = 60 * 60 * 24 * 365 * 3 // 3 years
const MASKER_MIN_TTL = 1

type MaskContext = {
    hogFunctionId: string
    hash: string
    increment: number
    ttl: number
    allowedExecutions: number
    threshold: number | null
}

type GenericHogInvocationWithMasker = CyclotronJobInvocation & {
    masker?: MaskContext
}

type MaskPipelineResult = Awaited<ReturnType<RedisV2['usePipeline']>>

function allowedExecutionsForResult(result: MaskPipelineResult, index: number, masker: MaskContext): number | null {
    const newValue: number | null = result ? result[index * 2][1] : null
    if (newValue === null) {
        return null
    }

    const oldValue = newValue - masker.increment
    if (masker.threshold) {
        return Math.floor((newValue - 1) / masker.threshold) - Math.floor((oldValue - 1) / masker.threshold)
    }
    return oldValue === 0 ? 1 : 0
}

function isHogFunctionInvocation(invocation: CyclotronJobInvocation): invocation is CyclotronJobInvocationHogFunction {
    return (invocation as CyclotronJobInvocationHogFunction).hogFunction !== undefined
}

function isHogFlowInvocation(invocation: CyclotronJobInvocation): invocation is CyclotronJobInvocationHogFlow {
    return (invocation as CyclotronJobInvocationHogFlow).hogFlow !== undefined
}

// Helper to extract masking config from different types
function extractMaskingConfig(invocation: CyclotronJobInvocation): HogFunctionMasking | null {
    if (isHogFunctionInvocation(invocation)) {
        return invocation.hogFunction.masking || null
    }

    if (isHogFlowInvocation(invocation)) {
        return invocation.hogFlow.trigger_masking || null
    }

    throw new Error('Unable to extract masking config from unknown invocation type')
}

function extractGlobals(invocation: CyclotronJobInvocation): Record<string, any> {
    if (isHogFunctionInvocation(invocation)) {
        return invocation.state.globals
    }
    if (isHogFlowInvocation(invocation)) {
        // For hog flows, we need to construct globals from the filter globals and event
        return {
            event: invocation.state?.event,
            person: invocation.person,
        }
    }

    throw new Error('Unable to extract globals from unknown invocation type')
}

// Helper to get entity ID for masking
function getEntityId(invocation: CyclotronJobInvocation): string {
    if (isHogFunctionInvocation(invocation)) {
        return invocation.hogFunction.id
    }
    if (isHogFlowInvocation(invocation)) {
        return invocation.hogFlow.id
    }
    return invocation.functionId
}

function getTtl(invocation: CyclotronJobInvocation, maskingConfig: HogFunctionMasking): number {
    const maxTtl = isHogFlowInvocation(invocation) ? MASKER_MAX_TTL_HOG_FLOW : MASKER_MAX_TTL_HOG_FUNCTION
    return Math.max(MASKER_MIN_TTL, Math.min(maxTtl, maskingConfig.ttl ?? maxTtl))
}

/**
 * HogMaskerService
 *
 * Responsible for determining if a function is "masked" or not based on the function configuration
 */

// Hog masker is meant to be done per batch
export class HogMaskerService {
    constructor(
        private redis: RedisV2,
        private redisMirror: RedisV2
    ) {}

    public async filterByMasking<T extends CyclotronJobInvocation>(
        invocations: T[]
    ): Promise<{
        masked: T[]
        notMasked: T[]
        /**
         * Undoes the Redis increments this call made. The claims are written before the
         * caller persists the invocations, so if that persistence fails, the caller must
         * release the claims — otherwise a replay of the same batch sees every invocation
         * as already-masked and silently drops it.
         *
         * Pass the invocations that did persist to keep their claims, so a replay does not run them twice.
         */
        release: (persisted?: CyclotronJobInvocation[]) => Promise<void>
    }> {
        const invocationsWithMasker: GenericHogInvocationWithMasker[] = [...invocations]
        const masks: Record<string, MaskContext> = {}

        // We find all functions/flows that have a mask and we load their masking from redis
        for (const item of invocationsWithMasker) {
            const maskingConfig = extractMaskingConfig(item)
            if (maskingConfig) {
                const globals = extractGlobals(item)

                // TODO: Catch errors
                const execHogResult = await execHog(maskingConfig.bytecode, {
                    globals,
                    timeout: 50,
                })

                if (!execHogResult.execResult?.result) {
                    continue
                }
                // What to do if it is null....

                const hash = createHash('md5')
                    .update(String(execHogResult.execResult.result))
                    .digest('hex')
                    .substring(0, 32)
                const entityId = getEntityId(item)
                const hashKey = `${entityId}:${hash}`
                masks[hashKey] = masks[hashKey] || {
                    hash,
                    hogFunctionId: entityId,
                    increment: 0,
                    ttl: getTtl(item, maskingConfig),
                    threshold: maskingConfig.threshold,
                    allowedExecutions: 0,
                }
                masks[hashKey]!.increment++
                item.masker = masks[hashKey]
            }
        }

        if (Object.keys(masks).length === 0) {
            return { masked: [], notMasked: invocations, release: async () => {} }
        }

        const buildPipeline = (pipeline: RedisClientPipeline): void => {
            Object.values(masks).forEach(({ hogFunctionId, hash, increment, ttl }) => {
                pipeline.incrby(`${REDIS_KEY_TOKENS}/${hogFunctionId}/${hash}`, increment)
                // @ts-expect-error - NX is not typed in ioredis
                pipeline.expire(`${REDIS_KEY_TOKENS}/${hogFunctionId}/${hash}`, ttl, 'NX')
            })
        }

        const maskContexts = Object.values(masks)
        // Track which stores the increments actually landed on, so release() only
        // decrements those — decrementing a store whose increment failed would push
        // its counters negative and over-allow future executions.
        let incrementedRedis: MaskPipelineResult = null
        let incrementedMirror: MaskPipelineResult = null
        const result = await dualRead(
            'hog-masker.filterByMasking',
            async () =>
                (incrementedRedis = await this.redis.usePipeline({ name: 'masker', failOpen: true }, buildPipeline)),
            async () =>
                (incrementedMirror = await this.redisMirror.usePipeline(
                    { name: 'masker-mirror', failOpen: true },
                    buildPipeline
                )),
            (primary, secondary) =>
                maskContexts.every(
                    (masker, index) =>
                        allowedExecutionsForResult(primary, index, masker) ===
                        allowedExecutionsForResult(secondary, index, masker)
                )
        )

        const release = async (persisted: CyclotronJobInvocation[] = []): Promise<void> => {
            const kept = new Set(persisted.map((item) => (item as GenericHogInvocationWithMasker).masker))
            const released = maskContexts.filter((masker) => !kept.has(masker))
            if (released.length === 0) {
                return
            }
            const buildReleasePipeline = (pipeline: RedisClientPipeline): void => {
                released.forEach(({ hogFunctionId, hash, increment }) => {
                    pipeline.incrby(`${REDIS_KEY_TOKENS}/${hogFunctionId}/${hash}`, -increment)
                })
            }
            await Promise.all([
                incrementedRedis
                    ? this.redis.usePipeline({ name: 'masker-release', failOpen: true }, buildReleasePipeline)
                    : null,
                incrementedMirror
                    ? this.redisMirror.usePipeline(
                          { name: 'masker-release-mirror', failOpen: true },
                          buildReleasePipeline
                      )
                    : null,
            ])
        }

        maskContexts.forEach((masker, index) => {
            const allowedExecutions = allowedExecutionsForResult(result, index, masker)
            if (allowedExecutions === null) {
                // We fail closed here as with a masking config the typical case will be not to send
                return
            }
            masker.allowedExecutions = allowedExecutions
        })

        return invocationsWithMasker.reduce(
            (acc, item) => {
                if (item.masker) {
                    if (item.masker.allowedExecutions > 0) {
                        item.masker.allowedExecutions--
                        acc.notMasked.push(item as T)
                    } else {
                        acc.masked.push(item as T)
                    }
                } else {
                    acc.notMasked.push(item as T)
                }
                return acc
            },
            { masked: [], notMasked: [], release } as {
                masked: T[]
                notMasked: T[]
                release: () => Promise<void>
            }
        )
    }
}