:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { createHash } from 'crypto'
import type RE2 from 're2'
import type { LogRecord } from '~/logs/log-record-avro'
import { type FilterGroupNode, matchFilterGroup } from './filter-group-match'
export const SAMPLING_DECISION_KEEP = 'keep' as const
export const SAMPLING_DECISION_DROP = 'drop' as const
export const SAMPLING_DECISION_SAMPLE_KEPT = 'sample_kept' as const
export const SAMPLING_DECISION_SAMPLE_DROPPED = 'sample_dropped' as const
export type SamplingDecision =
| typeof SAMPLING_DECISION_KEEP
| typeof SAMPLING_DECISION_DROP
| typeof SAMPLING_DECISION_SAMPLE_KEPT
| typeof SAMPLING_DECISION_SAMPLE_DROPPED
export type SeverityAction = { type: 'keep' } | { type: 'drop' } | { type: 'sample'; rate: number }
export type CompiledSamplingRule = {
id: string
ruleType: 'severity_sampling' | 'path_drop' | 'rate_limit'
scopeService: string | null
pathRegex: RE2 | null
pathDropPatterns: RE2[] | null
/** When set, path_drop regexes match only this attribute; null uses pathForMatching(). */
pathDropMatchAttributeKey: string | null
/**
* Universal filter group from the drop-rule UI. Evaluated per record alongside
* legacy `pathDropPatterns`: a record is dropped if EITHER matches. Null when
* the rule has no filter_group configured (legacy / patterns-only rules).
*/
filterGroup: FilterGroupNode | null
severityActions: [SeverityAction, SeverityAction, SeverityAction, SeverityAction]
alwaysKeep: {
statusGte: number | null
latencyMsGt: number | null
attributePredicates: { key: string; op: string; value?: string }[]
} | null
/**
* Set for rate_limit rules with valid config. Token-bucket on Redis; `costUnit` controls
* whether each record consumes one token (`'records'`) or its own `bytes_uncompressed`
* (`'bytes'`). `refillPerSecond` and `poolMax` are expressed in the matching unit.
*/
rateLimit: { refillPerSecond: number; poolMax: number; costUnit: 'records' | 'bytes' } | null
}
export type CompiledRuleSet = {
rules: CompiledSamplingRule[]
/** True when any compiled rule applies per-line Redis rate limiting. */
hasRateLimitRules: boolean
}
const SEV_ORD_DEBUG = 0
const SEV_ORD_INFO = 1
const SEV_ORD_WARN = 2
const SEV_ORD_ERROR = 3
export function severityOrdinalFromRecord(record: LogRecord): number {
const t = (record.severity_text || '').toLowerCase()
if (t === 'trace' || t === 'debug') {
return SEV_ORD_DEBUG
}
if (t === 'info') {
return SEV_ORD_INFO
}
if (t === 'warn' || t === 'warning') {
return SEV_ORD_WARN
}
if (t === 'error' || t === 'fatal') {
return SEV_ORD_ERROR
}
const n = record.severity_number
if (n == null) {
return SEV_ORD_INFO
}
if (n >= 17) {
return SEV_ORD_ERROR
}
if (n >= 13) {
return SEV_ORD_WARN
}
if (n >= 9) {
return SEV_ORD_INFO
}
return SEV_ORD_DEBUG
}
function hash01FromTraceId(traceId: Buffer | null): number {
if (!traceId || traceId.length === 0) {
return Math.random()
}
const h = createHash('sha256').update(traceId).digest()
return h.readUInt32BE(0) / 0xffffffff
}
function getAttribute(record: LogRecord, key: string): string | undefined {
const a = record.attributes
if (!a) {
return undefined
}
return a[key]
}
function pathForMatching(record: LogRecord): string {
const candidates = ['url.path', 'http.path', 'http.route', 'path']
for (const k of candidates) {
const v = getAttribute(record, k)
if (v) {
return v
}
}
return ''
}
function alwaysKeepMatches(rule: CompiledSamplingRule, record: LogRecord): boolean {
const ak = rule.alwaysKeep
if (!ak) {
return false
}
if (ak.statusGte != null) {
const codeStr = getAttribute(record, 'http.status_code') || getAttribute(record, 'http.response.status_code')
const code = codeStr ? parseInt(codeStr, 10) : NaN
if (!Number.isNaN(code) && code >= ak.statusGte) {
return true
}
}
if (ak.latencyMsGt != null) {
const latStr = getAttribute(record, 'http.server.duration_ms') || getAttribute(record, 'duration_ms')
const lat = latStr ? parseFloat(latStr) : NaN
if (!Number.isNaN(lat) && lat > ak.latencyMsGt) {
return true
}
}
for (const p of ak.attributePredicates) {
const v = getAttribute(record, p.key)
if (p.op === 'exists' && v !== undefined) {
return true
}
if (p.op === 'eq' && p.value !== undefined && v === p.value) {
return true
}
if (p.op === 'ne' && p.value !== undefined && v !== undefined && v !== p.value) {
return true
}
}
return false
}
/**
* A path_drop rule matches when EITHER the legacy regex patterns match the
* resolved path/attribute, OR the universal filter_group matches the record.
* Both shapes are supported during the transition off `config.patterns` —
* once all rules carry a filter_group, the patterns branch will be retired.
*/
function pathDropMatches(rule: CompiledSamplingRule, record: LogRecord): boolean {
if (rule.pathDropPatterns && rule.pathDropPatterns.length > 0) {
const p = rule.pathDropMatchAttributeKey
? (getAttribute(record, rule.pathDropMatchAttributeKey) ?? '')
: pathForMatching(record)
for (const rx of rule.pathDropPatterns) {
if (rx.test(p)) {
return true
}
}
}
if (rule.filterGroup && matchFilterGroup(rule.filterGroup, record)) {
return true
}
return false
}
function matchesScope(rule: CompiledSamplingRule, record: LogRecord): boolean {
if (rule.scopeService != null && rule.scopeService !== '') {
const sn = record.service_name || ''
if (sn !== rule.scopeService) {
return false
}
}
if (rule.pathRegex) {
const p = pathForMatching(record)
if (!rule.pathRegex.test(p)) {
return false
}
}
return true
}
export type EvaluateResult = {
decision: SamplingDecision
/** UUID of the first matching rule, if any */
ruleId: string | null
}
export type SamplingClassifyResult =
| { kind: 'resolved'; decision: SamplingDecision; ruleId: string | null }
| { kind: 'rate_limit'; ruleId: string }
export type RateLimitPendingByRule = Map<string, number[]>
/**
* Stateless rule walk for ingestion. Deterministic rules (`path_drop`, `severity_sampling`)
* always win over `rate_limit` regardless of priority order: a record that any deterministic
* rule resolves never charges the rate-limit bucket. The first applicable `rate_limit` match
* is remembered while the walk continues; it is only returned when no deterministic rule
* resolves the record. Otherwise matches `evaluateLogRecord`.
*/
export function classifySamplingRecord(teamRuleSet: CompiledRuleSet | null, record: LogRecord): SamplingClassifyResult {
if (!teamRuleSet || teamRuleSet.rules.length === 0) {
return { kind: 'resolved', decision: SAMPLING_DECISION_KEEP, ruleId: null }
}
const ord = severityOrdinalFromRecord(record)
let pendingRateLimitRuleId: string | null = null
for (const rule of teamRuleSet.rules) {
if (!matchesScope(rule, record)) {
continue
}
if (alwaysKeepMatches(rule, record)) {
return { kind: 'resolved', decision: SAMPLING_DECISION_KEEP, ruleId: rule.id }
}
if (rule.ruleType === 'path_drop') {
if (pathDropMatches(rule, record)) {
return { kind: 'resolved', decision: SAMPLING_DECISION_DROP, ruleId: rule.id }
}
continue
}
if (rule.ruleType === 'severity_sampling') {
const action = rule.severityActions[ord]
if (action.type === 'keep') {
return { kind: 'resolved', decision: SAMPLING_DECISION_KEEP, ruleId: rule.id }
}
if (action.type === 'drop') {
return { kind: 'resolved', decision: SAMPLING_DECISION_DROP, ruleId: rule.id }
}
const u = hash01FromTraceId(record.trace_id)
if (u < action.rate) {
return { kind: 'resolved', decision: SAMPLING_DECISION_SAMPLE_KEPT, ruleId: rule.id }
}
return { kind: 'resolved', decision: SAMPLING_DECISION_SAMPLE_DROPPED, ruleId: rule.id }
}
if (rule.ruleType === 'rate_limit') {
if (!rule.rateLimit) {
continue
}
// Mirror path_drop: filter_group must match if set. scope_service was
// already checked by matchesScope; this adds the modern scoping that
// the drop-rule UI now writes for rate_limit rules.
if (rule.filterGroup && !matchFilterGroup(rule.filterGroup, record)) {
continue
}
if (pendingRateLimitRuleId === null) {
pendingRateLimitRuleId = rule.id
}
continue
}
}
if (pendingRateLimitRuleId !== null) {
return { kind: 'rate_limit', ruleId: pendingRateLimitRuleId }
}
return { kind: 'resolved', decision: SAMPLING_DECISION_KEEP, ruleId: null }
}
export function evaluateLogRecord(teamRuleSet: CompiledRuleSet | null, record: LogRecord): EvaluateResult {
if (!teamRuleSet || teamRuleSet.rules.length === 0) {
return { decision: SAMPLING_DECISION_KEEP, ruleId: null }
}
const ord = severityOrdinalFromRecord(record)
for (const rule of teamRuleSet.rules) {
if (!matchesScope(rule, record)) {
continue
}
if (alwaysKeepMatches(rule, record)) {
return { decision: SAMPLING_DECISION_KEEP, ruleId: rule.id }
}
if (rule.ruleType === 'path_drop') {
if (pathDropMatches(rule, record)) {
return { decision: SAMPLING_DECISION_DROP, ruleId: rule.id }
}
continue
}
if (rule.ruleType === 'severity_sampling') {
const action = rule.severityActions[ord]
if (action.type === 'keep') {
return { decision: SAMPLING_DECISION_KEEP, ruleId: rule.id }
}
if (action.type === 'drop') {
return { decision: SAMPLING_DECISION_DROP, ruleId: rule.id }
}
const u = hash01FromTraceId(record.trace_id)
if (u < action.rate) {
return { decision: SAMPLING_DECISION_SAMPLE_KEPT, ruleId: rule.id }
}
return { decision: SAMPLING_DECISION_SAMPLE_DROPPED, ruleId: rule.id }
}
if (rule.ruleType === 'rate_limit') {
// Stateful in ingestion; stateless evaluation does not drop here.
continue
}
}
return { decision: SAMPLING_DECISION_KEEP, ruleId: null }
}