:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { createTrackedRE2 } from '~/common/utils/tracked-re2'
import { parseLogBodyForIngestion } from './log-body-parse'
export const PATTERN_VERSION = 6
/**
* Everything here shapes the emitted pattern, so it sits inside `PATTERN_VERSION`: two records may
* only be grouped by `pattern` when they carry the same version.
*
* These are constants rather than config for that reason. A per-pod override would let two pods stamp
* one version onto differently shaped patterns, which no consumer could detect. The shape-safe
* operational lever is `LOGS_PATTERN_MASKING_ENABLED_TEAMS`, which stops masking instead of changing it.
*
* `computeLogPattern` takes no caps argument, so there is no override path to reach at all. `readonly`
* states the intent to the compiler and `Object.freeze` holds it at runtime, because a caller reaching
* this object through an `any` would otherwise reshape every pattern under an unchanged version.
*/
export type PatternCaps = {
/** Ceiling on the body chars fed to the masker; longer bodies are cut first (CPU guard). */
readonly maxInputChars: number
/** Truncation applied to the masked pattern, after masking, so more real content survives the cut. */
readonly maxOutputChars: number
}
export const PATTERN_CAPS: PatternCaps = Object.freeze({
maxInputChars: 8192,
maxOutputChars: 1024,
})
export type MaskRuleName =
| 'timestamp'
| 'klogtime'
| 'clftime'
| 'ctime'
| 'httpdate'
| 'syslogtime'
| 'email'
| 'id'
| 'stripeid'
| 'snakeuuid'
| 'uuid'
| 'snakenum'
| 'host'
| 'hex0x'
| 'hex'
| 'ipv4'
| 'num'
export type MaskRule = {
name: MaskRuleName
pattern: string
replacement: string
}
const MONTH = '(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)'
const WEEKDAY = '(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun)'
const DAY_OF_MONTH = '(?:0?[1-9]|[12]\\d|3[01])'
const TIME_OF_DAY = '\\d{2}:\\d{2}:\\d{2}'
const YEAR = '(?:19|20)\\d{2}'
// `ctime` ends on an optional zone and a year, which is the same shape as an uppercase word and a
// count. Both fields are spelled out so the rule cannot take the second reading: a severity word is
// not a zone, and a count is not a year. Leaving either open merges an ERROR line with a WARN one,
// because the zone and the count both leave the pattern with the date.
//
// `httpdate` already enumerates its zone. A zone missing from this list costs grouping, not
// correctness: the line falls to `syslogtime`, which masks the date and leaves the zone and year.
const ZONE =
'(?:UT|UTC|GMT|Z|E[SD]T|C[SD]T|M[SD]T|P[SD]T|AK[SD]T|H[SD]T|A[SD]T|N[SD]T|CES?T|EES?T|WES?T|BST|IST|MSK|JST|KST|HKT|SGT|ICT|AE[SD]T|AC[SD]T|AWST|NZ[SD]T|SAST|WAT|CAT|EAT|BRT|ART|GST|PKT)'
export const MASK_RULES: readonly MaskRule[] = [
{
name: 'timestamp',
pattern: `\\b\\d{4}-\\d{2}-\\d{2}[T ]${TIME_OF_DAY}(?:[.,]\\d+)?(?:Z|[+-]\\d{2}:?\\d{2})?`,
replacement: '<TIMESTAMP>',
},
// klog / glog headers ("I0827 11:39:40.307946") carry no year and no separators, so the
// timestamp rule above misses them and `\b\d+` cannot reach the date either — there is no word
// boundary between the severity letter and the digits, so the date survives as a literal and
// the pattern changes every midnight.
//
// One rule per severity letter, rather than one rule with a lookbehind. The letter is content,
// not a variable, so it has to survive into the output; replacements are constant strings, so
// the only way to keep it is to consume it and write it back. Lookbehind and backreferences are
// both unavailable — RE2 lacks the first, and the rule-fire attribution in MASK_COMBINED_RE
// indexes capture groups positionally, so a rule may not add one.
//
// The letter also guards the match. Without it, `\d{4} \d{2}:\d{2}:\d{2}` would claim any count
// followed by a time of day.
//
// The date is spelled out as a real MMDD rather than `\d{4}`, because the letter alone is a weak
// guard: the rule reaches text that `\b\d+` cannot (there is no boundary between the letter and
// the digits), so a bare `\d{4}` masks any letter-prefixed 4-digit token that happens to precede
// a time — collapsing `E2024 10:20:30` and `E2025 10:20:30`, which are distinct today. A month of
// 01-12 and a day of 01-31 rejects those while accepting every date klog can print. The match
// cannot be anchored to the line start instead: Kubernetes CRI lines carry a container prefix
// ("<time> stderr F I0827 ..."), so the header is mid-string in the most common real source.
...(['I', 'W', 'E', 'F'] as const).map((letter) => ({
name: 'klogtime' as const,
pattern: `\\b${letter}(?:0[1-9]|1[0-2])(?:0[1-9]|[12]\\d|3[01]) ${TIME_OF_DAY}(?:\\.\\d+)?`,
replacement: `${letter}<KLOGTIME>`,
})),
// Date formats that spell the month or the weekday as a name. `\b\d+` masks the digits around
// the name but cannot touch the name itself, so the name survives into the pattern as a literal:
// one line splits into 12 patterns over a year, and into 7 over a week once a weekday is in it.
// Each rule consumes the whole date, so the name leaves with it.
//
// The name also guards the match: nothing here claims a bare number followed by a time of day.
//
// Order matters between these rules and is asserted by the sequential-chain corpus. A ctime line
// matches `ctime` and `syslogtime` at the same offset, and only `ctime` reaches the year. Listing
// it first is what makes a single pass and a rule-at-a-time chain agree: the single pass prefers
// the earlier alternative on a tie, and the chain runs `ctime` first.
{
name: 'clftime',
pattern: `\\b\\d{2}/${MONTH}/${YEAR}:${TIME_OF_DAY}(?: [+-]\\d{4})?`,
replacement: '<TIMESTAMP>',
},
{
name: 'ctime',
pattern: `\\b${WEEKDAY} ${MONTH} {1,2}${DAY_OF_MONTH} ${TIME_OF_DAY}(?: ${ZONE})? ${YEAR}`,
replacement: '<TIMESTAMP>',
},
{
name: 'httpdate',
pattern: `\\b${WEEKDAY}, ${DAY_OF_MONTH} ${MONTH} ${YEAR} ${TIME_OF_DAY}(?: GMT| UTC| [+-]\\d{4})?`,
replacement: '<TIMESTAMP>',
},
{
// The weekday is optional here as well as in `ctime`, because a line can carry one without
// carrying a year. Without it such a line falls to the month rule, which leaves the weekday
// behind as a literal and splits the line seven ways over a week.
name: 'syslogtime',
pattern: `\\b(?:${WEEKDAY} )?${MONTH} {1,2}${DAY_OF_MONTH} ${TIME_OF_DAY}`,
replacement: '<TIMESTAMP>',
},
// Listed ahead of `id`, which would otherwise take a local part shaped like `john_D2oe` and leave
// the domain to `host`.
{ name: 'email', pattern: '[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,}', replacement: '<EMAIL>' },
// Prefixed vendor ids (Stripe `cus_`/`sub_`, Clerk `user_`, ULID-shaped `org_`). The first segment
// needs an uppercase letter and a digit to stay off snake_case words; RE2 lacks lookahead, so both
// orders are spelled out. Hyphen-joined segments belong to the id, so a hyphenated token masks
// whole instead of leaving its tail behind as literals. Listed ahead of `uuid` so the sequential
// chain agrees with the single pass on a prefixed uppercase UUID.
{
name: 'id',
pattern:
'\\b[a-z]{2,10}_(?:[A-Za-z0-9]*[A-Z][A-Za-z0-9]*[0-9]|[A-Za-z0-9]*[0-9][A-Za-z0-9]*[A-Z])[A-Za-z0-9]*(?:-[A-Za-z0-9]+)*',
replacement: '<ID>',
},
// A digit-free Stripe id, which `id` cannot reach. Letters alone cannot tell an id from a camelCase
// name, so only Stripe's exact prefixes and 14 or 24 letter lengths match.
{
name: 'stripeid',
pattern: '\\b(?:cus|sub)_[A-Z][A-Za-z]{13}(?:[A-Za-z]{10})?\\b',
replacement: '<ID>',
},
// `\b\d+` cannot reach digits after `_`, because `_` is a word character. Runs under 4 digits stay
// literal so constants such as `aes_128` keep their meaning.
//
// `snakeuuid` sits ahead of `uuid` and `snakenum` because the leftmost match wins: otherwise
// `snakenum` takes digits that run into a UUID and leaves the rest of the UUID as literals.
{
name: 'snakeuuid',
pattern:
'_\\d{4,}(?:_\\d+)*_[0-9a-fA-F]*[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}',
replacement: '_<N>_<UUID>',
},
{
name: 'snakeuuid',
pattern: '_[0-9a-fA-F]*[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}',
replacement: '_<UUID>',
},
// The leading hex run takes characters glued to the front of a UUID, which `hex` or `num` would
// otherwise claim first and so leave the UUID's other groups behind as literals.
{
name: 'uuid',
pattern: '[0-9a-fA-F]*[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}',
replacement: '<UUID>',
},
{ name: 'snakenum', pattern: '_\\d{4,}(?:_\\d+)*\\b', replacement: '_<N>' },
// The suffix list carries no `so` or `sh`. Both are real TLDs, but the rule cannot tell a host
// from a filename, and shared object and shell script names reach it far more often than a
// Somali or Saint Helenian domain does. With them in the list, `libssl.so` and `deploy.sh` both
// mask to `<HOST>`, which merges lines that name different files.
{
name: 'host',
pattern:
'\\b(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?\\.){1,8}(?:ai|app|aws|bot|cloud|co|com|de|dev|eu|fr|gg|internal|io|jp|local|me|net|nl|org|tv|uk|us|xyz)\\b',
replacement: '<HOST>',
},
{ name: 'hex0x', pattern: '\\b0x[0-9a-fA-F]+\\b', replacement: '<HEX>' },
// The order of the rules below is what keeps a plain digit run on `num`. A whole word of digits
// reaches `num` before `hex` can see it, so an epoch, an id, or a byte count of 8 or more digits
// stays `<N>`, and `hex` takes only the runs that hold a letter. Splitting `num` in two is how
// "hex run containing a letter" reads without a lookahead, which RE2 does not have. `ipv4` leads
// so a dotted quad stays one match instead of four numbers.
{ name: 'ipv4', pattern: '\\b\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\b', replacement: '<IP>' },
{ name: 'num', pattern: '\\b\\d+\\b', replacement: '<N>' },
// Shortest hex run read as an identifier. Git short shas and container ids sit at 8 to 12 chars.
{ name: 'hex', pattern: '\\b[0-9a-fA-F]{8,}\\b', replacement: '<HEX>' },
{ name: 'num', pattern: '\\b\\d+', replacement: '<N>' },
]
export const JSON_ARRAY = '<JSON_ARRAY>'
export const KEY_SET_MAX_KEYS = 32
const MASK_COMBINED_RE = createTrackedRE2(
MASK_RULES.map((rule) => `(${rule.pattern})`).join('|'),
'g',
'log-pattern-mask:combined'
)
export type MaskResult = {
masked: string
ruleFires: number[]
}
/**
* One zeroed counter per rule in `MASK_RULES`, which `klogtime` spreads over four of.
*
* Every result carries a full-width vector, including the bodies that fire no rule at all. A shorter
* array would read as `undefined` to a caller that indexes by rule position, and reach a counter as
* `NaN`.
*/
export const zeroRuleFires = (): number[] => new Array(MASK_RULES.length).fill(0)
/** Masks `input` and adds its rule hits into `ruleFires`, so several strings can share one array. */
function maskInto(input: string, ruleFires: number[]): string {
return input.replace(MASK_COMBINED_RE, (...args: unknown[]): string => {
for (let i = 0; i < MASK_RULES.length; i++) {
if (args[i + 1] !== undefined) {
ruleFires[i]++
return MASK_RULES[i].replacement
}
}
return args[0] as string
})
}
export function maskString(input: string): MaskResult {
const ruleFires = zeroRuleFires()
return { masked: maskInto(input, ruleFires), ruleFires }
}
export type PatternBodyKind = 'empty' | 'plaintext' | 'json_object_or_array' | 'json_string' | 'primitive'
export type LogPatternResult = {
pattern: string
bodyKind: PatternBodyKind
inputCapped: boolean
maskedLength: number
jsonKeyCount?: number
ruleFires: number[]
}
export const MESSAGE_KEYS = ['message', 'msg', 'event'] as const
function extractJsonMessage(value: object, messageKeys: readonly string[]): string | null {
if (Array.isArray(value)) {
return null
}
for (const key of messageKeys) {
const candidate = (value as Record<string, unknown>)[key]
if (typeof candidate === 'string') {
return candidate
}
}
return null
}
const capOutput = (pattern: string): string =>
pattern.length > PATTERN_CAPS.maxOutputChars ? pattern.slice(0, PATTERN_CAPS.maxOutputChars) : pattern
/**
* Keys go through the same mask rules as a message, because a key set is just as often built from
* data: an object keyed by user id, host, or trace id otherwise emits one pattern per record, which
* is the shape the key set exists to collapse.
*
* Deduplicating after the mask is what bounds such an object to a single key, and doing it before
* the cap keeps the dropped-key count off the raw key count, because `+41` and `+42` are two
* patterns for one shape.
*/
function jsonKeySetPattern(rawKeys: readonly string[]): { pattern: string; ruleFires: number[] } {
const ruleFires = zeroRuleFires()
const masked = new Set<string>()
for (const key of rawKeys) {
masked.add(maskInto(key, ruleFires))
}
const keys = [...masked].sort()
const kept = keys.slice(0, KEY_SET_MAX_KEYS)
const overflow = keys.length - kept.length
return { pattern: `<JSON:${kept.join(',')}${overflow > 0 ? `,+${overflow}` : ''}>`, ruleFires }
}
export type PatternInputSelection =
| { kind: 'empty'; bodyKind: PatternBodyKind; inputCapped: boolean }
| { kind: 'mask'; input: string; bodyKind: PatternBodyKind; inputCapped: boolean }
| {
kind: 'pattern'
pattern: string
bodyKind: PatternBodyKind
inputCapped: boolean
ruleFires: number[]
jsonKeyCount?: number
}
export function selectPatternInput(
body: string | null | undefined,
messageKeys: readonly string[]
): PatternInputSelection {
if (body === null || body === undefined || body === '') {
return { kind: 'empty', bodyKind: 'empty', inputCapped: false }
}
const inputCapped = body.length > PATTERN_CAPS.maxInputChars
const cappedBody = inputCapped ? body.slice(0, PATTERN_CAPS.maxInputChars) : body
const parsed = parseLogBodyForIngestion(cappedBody)
const bodyKind: PatternBodyKind =
parsed.kind === 'json_primitive' ? 'primitive' : parsed.kind === 'invalid_json' ? 'plaintext' : parsed.kind
switch (parsed.kind) {
case 'empty':
return { kind: 'mask', input: '', bodyKind, inputCapped }
case 'json_object_or_array': {
const message = extractJsonMessage(parsed.value, messageKeys)
if (message === null) {
if (Array.isArray(parsed.value)) {
return { kind: 'pattern', pattern: JSON_ARRAY, bodyKind, inputCapped, ruleFires: zeroRuleFires() }
}
const rawKeys = Object.keys(parsed.value)
const keySet = jsonKeySetPattern(rawKeys)
return {
kind: 'pattern',
pattern: keySet.pattern,
bodyKind,
inputCapped,
ruleFires: keySet.ruleFires,
jsonKeyCount: rawKeys.length,
}
}
return { kind: 'mask', input: message, bodyKind, inputCapped }
}
case 'json_string':
return { kind: 'mask', input: parsed.value, bodyKind, inputCapped }
case 'json_primitive':
return { kind: 'mask', input: cappedBody, bodyKind, inputCapped }
case 'invalid_json':
return { kind: 'mask', input: parsed.raw, bodyKind, inputCapped }
}
}
export type MaskedPattern = {
pattern: string
maskedLength: number
ruleFires: number[]
}
export function computeLogPattern(input: string): MaskedPattern {
const { masked, ruleFires } = maskString(input)
return { pattern: capOutput(masked), maskedLength: masked.length, ruleFires }
}
export function buildLogPattern(body: string | null | undefined, messageKeys: readonly string[]): LogPatternResult {
const selection = selectPatternInput(body, messageKeys)
const { bodyKind, inputCapped } = selection
if (selection.kind === 'empty') {
return { pattern: '', bodyKind, inputCapped, maskedLength: 0, ruleFires: zeroRuleFires() }
}
if (selection.kind === 'pattern') {
return {
pattern: capOutput(selection.pattern),
bodyKind,
inputCapped,
maskedLength: selection.pattern.length,
...(selection.jsonKeyCount === undefined ? {} : { jsonKeyCount: selection.jsonKeyCount }),
ruleFires: selection.ruleFires,
}
}
return { ...computeLogPattern(selection.input), bodyKind, inputCapped }
}