Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

13.5 KB
Raw Blame
import { createTrackedRE2 } from '~/common/utils/tracked-re2'

import type { CompiledRuleSet, CompiledSamplingRule, SeverityAction } from './evaluate'
import { type FilterGroupNode, MAX_FILTER_GROUP_DEPTH } from './filter-group-match'
import { type PropertyFilterLeaf, compileLeafRegex } from './property-filter-match'

/**
 * Bound on the total node count (groups + leaves) of a filter_group tree. Depth
 * alone does not bound per-record work: a single AND with thousands of sibling
 * leaves passes MAX_FILTER_GROUP_DEPTH but costs O(leaves) per log line. Kept
 * in sync with `MAX_FILTER_GROUP_NODES` in `products/logs/backend/presentation/filter_group_validation.py`.
 */
export const MAX_FILTER_GROUP_NODES = 256

/**
 * Aggregate cap on leaf filter values (a value array counts as its length).
 * Kept in sync with `MAX_FILTER_GROUP_LEAF_VALUES` in
 * `products/logs/backend/presentation/filter_group_validation.py`.
 */
export const MAX_FILTER_GROUP_LEAF_VALUES = 256

export type SamplingRuleRow = {
    id: string
    rule_type: string
    scope_service: string | null
    scope_path_pattern: string | null
    scope_attribute_filters: unknown
    config: Record<string, unknown>
    /** Row version from DB; used by cache watermark only, ignored by compileRuleSet. */
    version?: number
}

const defaultSeverityActions = (): [SeverityAction, SeverityAction, SeverityAction, SeverityAction] => [
    { type: 'keep' },
    { type: 'keep' },
    { type: 'keep' },
    { type: 'keep' },
]

function parseSeverityActions(raw: unknown): [SeverityAction, SeverityAction, SeverityAction, SeverityAction] {
    const out = defaultSeverityActions()
    if (!raw || typeof raw !== 'object') {
        return out
    }
    const actions = (raw as { actions?: Record<string, unknown> }).actions
    if (!actions || typeof actions !== 'object') {
        return out
    }
    const levels = ['DEBUG', 'INFO', 'WARN', 'ERROR'] as const
    for (let i = 0; i < levels.length; i++) {
        const key = levels[i]
        const a = actions[key] as Record<string, unknown> | undefined
        if (!a || typeof a.type !== 'string') {
            continue
        }
        if (a.type === 'drop') {
            out[i] = { type: 'drop' }
        } else if (a.type === 'sample' && typeof a.rate === 'number') {
            out[i] = { type: 'sample', rate: Math.max(0, Math.min(1, a.rate)) }
        } else {
            out[i] = { type: 'keep' }
        }
    }
    return out
}

const MAX_LOGS_PER_SECOND = 1_000_000
const MAX_BURST_LOGS = 10_000_000
const MAX_KB_PER_SECOND = 1_000_000
const MAX_BURST_KB = 10_000_000
// Decimal (SI) KB: the drop-rule form (UNIT_TO_KB_PER_S), the sparkline preview's
// threshold line (KB/s × 1000), and the API validator ("1000000 = 1 GB/s") all use
// 1 KB = 1000 bytes. The bucket must charge in the same unit, otherwise every cap
// silently enforces ~2.4% above its label.
const BYTES_PER_KB = 1000

function parseRateLimitFromConfig(
    config: Record<string, unknown>
): { refillPerSecond: number; poolMax: number; costUnit: 'records' | 'bytes' } | null {
    // KB-mode takes precedence when both fields are set. The API validator rejects that
    // case at write time, but the ingestion path stays robust to legacy or hand-crafted rows.
    const kbps = config.kb_per_second
    if (typeof kbps === 'number' && Number.isFinite(kbps) && kbps >= 1 && kbps <= MAX_KB_PER_SECOND) {
        const refillKb = Math.floor(kbps)
        const burstRaw = config.burst_kb
        const burstKb =
            typeof burstRaw === 'number' && Number.isFinite(burstRaw) && burstRaw >= refillKb
                ? Math.min(Math.floor(burstRaw), MAX_BURST_KB)
                : Math.min(refillKb * 3, MAX_BURST_KB)
        return {
            refillPerSecond: refillKb * BYTES_PER_KB,
            poolMax: burstKb * BYTES_PER_KB,
            costUnit: 'bytes',
        }
    }
    const lps = config.logs_per_second
    if (typeof lps !== 'number' || !Number.isFinite(lps) || lps < 1 || lps > MAX_LOGS_PER_SECOND) {
        return null
    }
    const refill = Math.floor(lps)
    const burstRaw = config.burst_logs
    const poolMax =
        typeof burstRaw === 'number' && Number.isFinite(burstRaw) && burstRaw >= refill
            ? Math.min(Math.floor(burstRaw), MAX_BURST_LOGS)
            : Math.min(refill * 3, MAX_BURST_LOGS)
    return { refillPerSecond: refill, poolMax, costUnit: 'records' }
}

/**
 * The drop-rules UI writes the inner group wrapped in another AND envelope:
 *   { type: AND, values: [ { type: AND|OR, values: [<leaves>] } ] }
 * Earlier shapes may have stored the bare inner group. Accept either; reject
 * anything that doesn't look like a group.
 */
export function parseFilterGroup(raw: unknown): FilterGroupNode | null {
    if (!raw || typeof raw !== 'object') {
        return null
    }
    const candidate = raw as { type?: unknown; values?: unknown }
    if (!Array.isArray(candidate.values) || (candidate.type !== 'AND' && candidate.type !== 'OR')) {
        return null
    }
    let parsed: FilterGroupNode = candidate as FilterGroupNode
    // If the outer envelope contains a single inner group, unwrap it.
    if (candidate.values.length === 1) {
        const inner = candidate.values[0] as { type?: unknown; values?: unknown } | null
        if (
            inner &&
            typeof inner === 'object' &&
            Array.isArray(inner.values) &&
            (inner.type === 'AND' || inner.type === 'OR')
        ) {
            parsed = inner as FilterGroupNode
        }
    }
    // Reject pathologically deep trees at compile time so the worker hot path
    // never recurses past MAX_FILTER_GROUP_DEPTH. Same bound is enforced in
    // sampling_api.py's Pydantic validator at write time; this is defense in
    // depth for rows that predate the validator.
    if (filterGroupDepth(parsed, 0) > MAX_FILTER_GROUP_DEPTH) {
        return null
    }
    // Also bound total breadth — a flat AND with thousands of sibling leaves
    // passes the depth check but costs O(leaves) per record. Matches the
    // server-side check in sampling_api.py and protects rows that predate it.
    if (filterGroupNodeCount(parsed) > MAX_FILTER_GROUP_NODES) {
        return null
    }
    // And bound leaf VALUES — one `exact`/`in` leaf carrying a huge value array
    // is a single node but costs O(values) per record in matchExact. Matches
    // MAX_FILTER_GROUP_LEAF_VALUES in filter_group_validation.py.
    if (filterGroupLeafValueCount(parsed) > MAX_FILTER_GROUP_LEAF_VALUES) {
        return null
    }
    // Walk the tree once and stamp pre-compiled regex onto each regex leaf so
    // the per-record hot path doesn't allocate a fresh `RegExp` per match.
    // Legacy `pathDropPatterns` already follow this pattern; this brings the
    // filter-group path in line.
    compileRegexLeavesInPlace(parsed)
    return parsed
}

function filterGroupLeafValueCount(node: FilterGroupNode | PropertyFilterLeaf): number {
    const maybe = node as { type?: unknown; values?: unknown; value?: unknown }
    if (!Array.isArray(maybe.values) || (maybe.type !== 'AND' && maybe.type !== 'OR')) {
        if (Array.isArray(maybe.value)) {
            return maybe.value.length
        }
        return maybe.value === undefined || maybe.value === null ? 0 : 1
    }
    let total = 0
    for (const child of maybe.values as Array<FilterGroupNode | PropertyFilterLeaf>) {
        total += filterGroupLeafValueCount(child)
        if (total > MAX_FILTER_GROUP_LEAF_VALUES) {
            return total
        }
    }
    return total
}

function filterGroupNodeCount(node: FilterGroupNode | PropertyFilterLeaf): number {
    const maybe = node as { type?: unknown; values?: unknown }
    if (!Array.isArray(maybe.values) || (maybe.type !== 'AND' && maybe.type !== 'OR')) {
        return 1
    }
    let total = 1
    for (const child of maybe.values as Array<FilterGroupNode | PropertyFilterLeaf>) {
        total += filterGroupNodeCount(child)
        if (total > MAX_FILTER_GROUP_NODES) {
            return total
        }
    }
    return total
}

function filterGroupDepth(node: FilterGroupNode | PropertyFilterLeaf, depth: number): number {
    const maybe = node as { type?: unknown; values?: unknown }
    if (!Array.isArray(maybe.values) || (maybe.type !== 'AND' && maybe.type !== 'OR')) {
        return depth
    }
    let maxChild = depth
    for (const child of maybe.values as Array<FilterGroupNode | PropertyFilterLeaf>) {
        const d = filterGroupDepth(child, depth + 1)
        if (d > maxChild) {
            maxChild = d
        }
    }
    return maxChild
}

function compileRegexLeavesInPlace(node: FilterGroupNode | PropertyFilterLeaf): void {
    const maybe = node as { type?: unknown; values?: unknown }
    if (Array.isArray(maybe.values) && (maybe.type === 'AND' || maybe.type === 'OR')) {
        for (const child of maybe.values as Array<FilterGroupNode | PropertyFilterLeaf>) {
            compileRegexLeavesInPlace(child)
        }
        return
    }
    const leaf = node as PropertyFilterLeaf
    if (leaf.operator === 'regex' || leaf.operator === 'not_regex') {
        leaf._compiledRegex = leaf.value == null ? null : compileLeafRegex(leaf.value)
    }
}

function parseAlwaysKeep(config: Record<string, unknown>): CompiledSamplingRule['alwaysKeep'] {
    const ak = config.always_keep as Record<string, unknown> | undefined
    if (!ak || typeof ak !== 'object') {
        return null
    }
    const statusGte = typeof ak.status_gte === 'number' ? ak.status_gte : null
    const latencyMsGt = typeof ak.latency_ms_gt === 'number' ? ak.latency_ms_gt : null
    const preds = Array.isArray(ak.attribute_predicates) ? (ak.attribute_predicates as any[]) : []
    const attributePredicates = preds
        .filter((p) => p && typeof p.key === 'string' && typeof p.op === 'string')
        .map((p) => ({
            key: p.key as string,
            op: p.op as string,
            value: typeof p.value === 'string' ? p.value : undefined,
        }))
    if (statusGte == null && latencyMsGt == null && attributePredicates.length === 0) {
        return null
    }
    return { statusGte, latencyMsGt, attributePredicates }
}

export function compileRuleSet(rows: SamplingRuleRow[]): CompiledRuleSet {
    const rules: CompiledSamplingRule[] = []
    let hasRateLimitRules = false
    for (const row of rows) {
        // RE2 has linear-time matching; native RegExp here would expose the ingestion
        // worker to catastrophic-backtracking ReDoS from any admin-authored pattern.
        // Same engine choice as the new filter-group regex leaves and the rest of
        // `nodejs/src/cdp/` regex sites.
        let pathRegex: CompiledSamplingRule['pathRegex'] = null
        if (row.scope_path_pattern) {
            try {
                pathRegex = createTrackedRE2(row.scope_path_pattern, undefined, 'logs-sampling:scope-path')
            } catch {
                // Treat invalid / RE2-rejected patterns as match-nothing so the rule
                // never accidentally over-scopes. Mirrors prior `/^$/` sentinel.
                pathRegex = createTrackedRE2('^$', undefined, 'logs-sampling:scope-path-fallback')
            }
        }
        let pathDropPatterns: CompiledSamplingRule['pathDropPatterns'] = null
        let pathDropMatchAttributeKey: string | null = null
        let filterGroup: FilterGroupNode | null = null
        if (row.rule_type === 'path_drop') {
            // Array.isArray, not truthiness: a corrupt string value would otherwise be
            // iterated per character into single-char regexes that match nearly everything.
            const patterns = Array.isArray(row.config.patterns) ? (row.config.patterns as unknown[]) : []
            pathDropPatterns = []
            for (const p of patterns) {
                if (typeof p !== 'string') {
                    continue
                }
                try {
                    pathDropPatterns.push(createTrackedRE2(p, undefined, 'logs-sampling:path-drop-pattern'))
                } catch {
                    /* skip invalid (incl. RE2-rejected lookahead/backreference patterns) */
                }
            }
            const mak = row.config.match_attribute_key
            if (typeof mak === 'string') {
                const t = mak.trim()
                pathDropMatchAttributeKey = t === '' ? null : t
            }
            filterGroup = parseFilterGroup(row.config.filter_group)
        }
        const rt = row.rule_type as CompiledSamplingRule['ruleType']
        const ruleType: CompiledSamplingRule['ruleType'] =
            rt === 'path_drop' || rt === 'rate_limit' || rt === 'severity_sampling' ? rt : 'path_drop'
        let rateLimit: CompiledSamplingRule['rateLimit'] = null
        if (ruleType === 'rate_limit') {
            rateLimit = parseRateLimitFromConfig(row.config ?? {})
            if (rateLimit) {
                hasRateLimitRules = true
            }
            // rate_limit rules also accept config.filter_group as a universal scope (the
            // drop-rule UI now writes service-scoping there instead of scope_service).
            // The path_drop branch above already parsed any filter_group on path_drop rows;
            // do the same parse for rate_limit so the evaluator can honor it.
            filterGroup = parseFilterGroup(row.config?.filter_group)
        }
        rules.push({
            id: row.id,
            ruleType,
            scopeService: row.scope_service,
            pathRegex,
            pathDropPatterns,
            pathDropMatchAttributeKey,
            filterGroup,
            severityActions: parseSeverityActions(row.config),
            alwaysKeep: parseAlwaysKeep(row.config),
            rateLimit,
        })
    }
    return { rules, hasRateLimitRules }
}