:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'fs'
import { join } from 'path'
import { parseJSON } from '~/common/utils/json-parse'
import { decodeLogAttributeValue, encodeLogAttributeValue } from '../attribute-value'
import { InternalLogsVerifier, signInternalLogs } from '../internal-logs'
import { parseLogBodyForIngestion } from '../log-body-parse'
import { MESSAGE_KEYS, buildLogPattern } from '../log-pattern-mask'
import { scrubPlainStringWithStats } from '../log-pii-scrub'
import { type LogRecord, transformDecodedLogRecordsInPlace } from '../log-record-avro'
import { type MetricRuleRow, compileMetricRules } from '../metrics-rules/compile-metric-rules'
import { withDefaultMetricRules } from '../metrics-rules/default-metric-rules'
import { buildMetricRulesOtlpPayload } from '../metrics-rules/otlp-payload'
import { createBatchTallies, tallyRecords } from '../metrics-rules/tally'
import { measureContentWeights } from '../pipeline/log-processing-pipeline'
import { type RetentionRuleRow, compileRetentionRuleSet } from '../retention/compile-retention-rules'
import { evaluateRetentionDays } from '../retention/evaluate-retention'
import { type SamplingRuleRow, compileRuleSet } from '../sampling/compile-rules'
import { evaluateLogRecord } from '../sampling/evaluate'
import { canonicalizeAvroValue } from './parity-harness'
// Recorded with `LOGS_PARITY_RECORD=1`. Inputs target what a port gets wrong: UTF-16 slicing, key order, number format, `Date.parse`.
const DIR = join(__dirname, 'fixtures', 'modules')
const RECORD = process.env.LOGS_PARITY_RECORD === '1'
const NOW_MS = Date.UTC(2025, 0, 15, 12, 37, 13, 250)
type Case = { input: unknown; output: unknown }
const STRINGS: string[] = [
'',
'plain text',
'user [email protected] logged in from 192.168.1.20 and 2001:db8::1',
'card 4111 1111 1111 1111 exp 12/29, 4111-1111-1111-1111, 5500000000000004',
'Authorization: Bearer abcdefghijklmnopqrstuvwxyz0123456789 token=sk_live_FAKEFAKEFAKE1234 password=hunter2',
'phone +1 (555) 123-4567 ssn 123-45-6789 iban GB82WEST12345698765432',
'aws AKIAIOSFODNN7EXAMPLE jwt eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxIn0.abc123def456',
'request 3fa85f64-5717-4562-b3fc-2c963f66afa6 took 12ms on host ip-10-0-0-1 at 2025-01-15T10:00:00.123Z',
'GET /api/users/12345/orders/987?page=2&token=abc 200 34ms',
'hex 0xdeadbeef sha 9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08',
'emoji 🚀🔥✓ and accents héllo wörld and CJK 日本語テキスト',
'🚀'.repeat(300),
'é'.repeat(5000),
'x'.repeat(70_000),
' leading and trailing whitespace \t\n',
'mixed\r\nline\nendings\rhere',
'{"msg": "json message 123", "level": "info", "user_id": 42}',
'{"message": {"nested": "object"}, "msg": "fallback 7"}',
'{"event": "user.signup", "id": "abc-123"}',
'{"text": "custom key 99"}',
'["array", "body", 1, 2]',
'"a json string 42"',
'12345.678',
'null',
'true',
'{"a": 1, "1": "int key", "b": {"c": [1, 2, {"d": null}]}, "0": "zero"}',
'{"big": 12345678901234567890, "small": 1e-7, "neg": -0, "exp": 1e21, "f": 0.30000000000000004, "i": 1.0}',
'{"__proto__": {"x": 1}, "constructor": "c", "toString": "t", "hasOwnProperty": "h"}',
'{"dup": 1, "dup": 2}',
'{"deep": ' + '['.repeat(140) + '1' + ']'.repeat(140) + '}',
JSON.stringify(Object.fromEntries(Array.from({ length: 70 }, (_, i) => [`k${i}`, i]))),
JSON.stringify({ arr: Array.from({ length: 80 }, (_, i) => i) }),
JSON.stringify({ arr: Array.from({ length: 12_000 }, () => 1) }),
JSON.stringify(Object.fromEntries(Array.from({ length: 60 }, (_, i) => [String(59 - i), i]))),
JSON.stringify({ 'dotted.key': { nested: 'v' }, dotted: { key: 'collision' } }),
JSON.stringify({ s: 'escaped \\n newline and "quotes" and \t tabs and \u0000 nul', u: '
' }),
JSON.stringify({ long: 'y'.repeat(70_000) }),
'{not valid json',
'{"trailing": "comma",}',
"{'single': 'quotes'}",
'{"bom": true}',
'<html><body>error 500 at /checkout</body></html>',
'ERROR [2025-01-15 10:00:00,123] com.example.Service - failed id=987654 [email protected]',
'10.0.0.1 - - [15/Jan/2025:10:00:00 +0000] "GET /index.html HTTP/1.1" 200 1043',
]
const DATE_STRINGS: (string | undefined)[] = [
undefined,
'',
'2025-01-15T12:00:00.000Z',
'2025-01-15T12:00:00Z',
'2025-01-15T12:00:00.123456Z',
'2025-01-15T12:00:00+02:00',
'2025-01-15T12:00:00',
'2025-01-15 12:00:00',
'2025-01-15',
'2025-1-5',
'Jan 15 2025 12:00:00 GMT',
'Wed, 15 Jan 2025 12:00:00 GMT',
'15 January 2025',
'1736942400',
'1736942400000',
'not a date',
'2025-02-30T00:00:00Z',
'2025-01-15T25:00:00Z',
'+002025-01-15T12:00:00Z',
' 2025-01-15T12:00:00Z ',
]
const RECORD_BODIES = STRINGS.filter((body) => body.length < 6000)
class Rng {
constructor(private state: number) {}
next(): number {
this.state = (this.state + 0x6d2b79f5) | 0
let t = this.state
t = Math.imul(t ^ (t >>> 15), t | 1)
t ^= t + Math.imul(t ^ (t >>> 7), t | 61)
return ((t ^ (t >>> 14)) >>> 0) / 4294967296
}
int(n: number): number {
return Math.floor(this.next() * n)
}
pick<T>(items: readonly T[]): T {
return items[this.int(items.length)]
}
}
const jsonAttr = (value: unknown): string => JSON.stringify(value)
function makeRecord(rng: Rng, i: number): LogRecord {
const attributes: Record<string, string> = {}
for (let a = 0; a < rng.int(7); a++) {
attributes[rng.pick(['http.method', 'url.path', 'user.email', 'count', 'payload', 'k', '1', 'level', 'ms'])] =
rng.pick([
jsonAttr('GET'),
jsonAttr('/healthz'),
jsonAttr('/api/v1/items'),
jsonAttr('[email protected]'),
jsonAttr(42),
jsonAttr(-3.5),
jsonAttr('12.5'),
jsonAttr(true),
jsonAttr({ nested: { json: 'payload', n: 1 } }),
jsonAttr(JSON.stringify({ stringified: 'json', arr: [1, 2] })),
'raw-unquoted',
jsonAttr('🚀'.repeat(200)),
jsonAttr(''),
])
}
return {
uuid: `uuid-${i}`,
trace_id: Buffer.from(Array.from({ length: 16 }, () => rng.int(256))),
span_id: rng.int(2) ? Buffer.from(Array.from({ length: 8 }, () => rng.int(256))) : null,
trace_flags: null,
timestamp: rng.int(4) === 0 ? null : (NOW_MS - rng.int(30 * 60 * 1000)) * 1000,
observed_timestamp: NOW_MS * 1000,
// Oversized bodies stay in the string vectors and two normalize cases, which keeps these files small.
body: rng.pick(RECORD_BODIES),
severity_text: rng.pick(['DEBUG', 'INFO', 'WARN', 'ERROR', 'info', 'warning', 'fatal', 'trace', '', null]),
severity_number: rng.pick([1, 5, 9, 13, 17, 21, 0, null]),
service_name: rng.pick(['api', 'worker', 'web', null]),
resource_attributes: rng.int(5)
? { 'service.name': jsonAttr(rng.pick(['api', 'worker'])), 'k8s.pod.name': jsonAttr(`pod-${rng.int(3)}`) }
: null,
instrumentation_scope: rng.pick([null, '[email protected]']),
event_name: rng.pick([null, 'request']),
attributes: rng.int(10) ? attributes : null,
bytes_uncompressed: rng.int(4) ? 50 + rng.int(800) : null,
}
}
/** The JSON form of a record in the vectors: bytes as `{ $bytes: hex }`, map key order kept. */
function recordToJson(record: LogRecord): unknown {
return Object.fromEntries(
Object.entries(record).map(([key, value]) => [
key,
Buffer.isBuffer(value) ? { $bytes: value.toString('hex') } : value,
])
)
}
const leaf = (key: string, operator: string, value: unknown, type = 'log_attribute') => ({ key, operator, value, type })
const group = (type: 'AND' | 'OR', ...values: unknown[]) => ({ type: 'AND', values: [{ type, values }] })
const FILTER_GROUPS: unknown[] = [
null,
{ type: 'XOR', values: [] },
group('AND', leaf('severity_text', 'exact', 'ERROR', 'log')),
group('AND', leaf('severity_level', 'exact', ['error', 'ERROR'])),
group('OR', leaf('user.email', 'icontains', 'EXAMPLE.COM'), leaf('count', 'gt', 10)),
group('AND', leaf('count', 'lte', '42'), leaf('count', 'gte', 42)),
group('AND', leaf('message', 'regex', '^(GET|POST) /api', 'log')),
group('AND', leaf('message', 'not_regex', '(?=lookahead)', 'log')),
group('AND', leaf('message', 'regex', 'HEALTHZ|token=', 'log')),
group('AND', leaf('url.path', 'starts_with', '/api'), leaf('http.method', 'is_not', 'POST')),
group('AND', leaf('url.path', 'ends_with', 'z')),
group('AND', leaf('k', 'is_set', null)),
group('AND', leaf('k', 'is_not_set', null)),
group('AND', leaf('service.name', 'exact', 'worker', 'log_resource_attribute')),
group('AND', leaf('service_name', 'in', ['api', 'web'])),
group('AND', leaf('k8s.pod.name', 'not_in', ['pod-0'], 'log_resource_attribute')),
group('AND', leaf('payload', 'not_icontains', 'json')),
group('AND', leaf('ms', 'unknown_operator', 'x')),
group(
'OR',
group('AND', leaf('level', 'exact', 'info'), leaf('count', 'lt', 50)),
leaf('1', 'exact', 'raw-unquoted')
),
]
function buildCases(): Record<string, Case[]> {
const rng = new Rng(42)
const records = Array.from({ length: 160 }, (_, i) => makeRecord(rng, i))
const out: Record<string, Case[]> = {}
out.body_parse = STRINGS.map((body) => ({ input: body, output: parseLogBodyForIngestion(body) }))
out.pii_scrub = STRINGS.map((input) => ({ input, output: scrubPlainStringWithStats(input) }))
out.attribute_value = [...STRINGS, ...records.flatMap((r) => Object.values(r.attributes ?? {}))].map((input) => ({
input,
output: { decoded: decodeLogAttributeValue(input), encoded: encodeLogAttributeValue(input) },
}))
out.pattern = [MESSAGE_KEYS, ['text', 'nope'], []].flatMap((keys) =>
STRINGS.map((body) => ({ input: { body, keys }, output: buildLogPattern(body, keys) }))
)
out.date_parse = DATE_STRINGS.map((input) => ({ input: input ?? null, output: Date.parse(input ?? '') }))
const settingsVariants = [
{ json_parse_logs: true },
{ pii_scrub_logs: true },
{ json_parse_logs: true, pii_scrub_logs: true },
{ json_parse_logs_attribute_key: 'payload' },
{ json_parse_logs: true, pii_scrub_logs: true, json_parse_logs_attribute_key: 'payload' },
]
out.normalize = []
const oversized = STRINGS.filter((body) => body.length >= 6000).map((body, i) => ({ ...records[i], body }))
for (const settings of settingsVariants.slice(2)) {
for (const input of oversized) {
out.normalize.push({
input: { settings, record: recordToJson(input) },
output: { ...input, attributes: input.attributes ? { ...input.attributes } : null },
})
}
}
for (const settings of settingsVariants) {
for (let i = 0; i < 40; i++) {
const input = records[(i * 7 + settingsVariants.indexOf(settings)) % records.length]
const copy: LogRecord = { ...input, attributes: input.attributes ? { ...input.attributes } : null }
// Synchronous body: the instrumented helpers resolve immediately.
out.normalize.push({ input: { settings, record: recordToJson(input) }, output: copy })
}
}
const samplingRuleSets: unknown[][] = [
[
{
id: 's1',
rule_type: 'severity_sampling',
scope_service: null,
scope_path_pattern: null,
scope_attribute_filters: [],
config: {
actions: {
DEBUG: { type: 'drop' },
INFO: { type: 'sample', rate: 0.5 },
WARN: { type: 'sample', rate: 1.5 },
},
},
},
],
[
{
id: 'p1',
rule_type: 'path_drop',
scope_service: 'api',
scope_path_pattern: null,
scope_attribute_filters: [],
config: { patterns: ['/healthz', '^/api/v[0-9]+'], match_attribute_key: 'url.path' },
},
],
[
{
id: 'p2',
rule_type: 'path_drop',
scope_service: null,
scope_path_pattern: '/api/*',
scope_attribute_filters: [],
config: { patterns: ['GET'], match_attribute_key: 'http.method', filter_group: FILTER_GROUPS[4] },
},
],
[
{
id: 'p3',
rule_type: 'path_drop',
scope_service: null,
scope_path_pattern: null,
scope_attribute_filters: [{ key: 'level', value: 'info' }],
config: { patterns: 'notarray' },
},
],
[
{
id: 'k1',
rule_type: 'severity_sampling',
scope_service: null,
scope_path_pattern: null,
scope_attribute_filters: [],
config: {
actions: { ERROR: { type: 'drop' } },
always_keep: {
status_gte: 500,
latency_ms_gt: 1000,
predicates: [{ key: 'k', op: 'eq', value: 'x' }],
},
},
},
],
[
{
id: 'u1',
rule_type: 'mystery',
scope_service: null,
scope_path_pattern: null,
scope_attribute_filters: [],
config: {},
},
],
]
out.sampling_evaluate = samplingRuleSets.flatMap((rows) => {
const ruleSet = compileRuleSet(rows as SamplingRuleRow[])
return records.slice(0, 60).map((record) => ({
input: { rules: rows, record: recordToJson(record) },
output: evaluateLogRecord(ruleSet, record),
}))
})
out.retention_evaluate = FILTER_GROUPS.flatMap((filterGroup, g) => {
const rows: RetentionRuleRow[] = [
{ id: `r${g}`, config: { retention_days: 90, filter_group: filterGroup } },
{
id: 'fallback',
config: { retention_days: 30, filter_group: group('AND', leaf('service_name', 'is_set', null)) },
},
]
const ruleSet = compileRetentionRuleSet(rows)
return records.slice(0, 25).map((record) => ({
input: { rules: rows, record: recordToJson(record) },
output: evaluateRetentionDays(ruleSet, record),
}))
})
const metricRuleSets: MetricRuleRow[][] = [
[{ id: 'm1', metric_name: 'count_all', filter_group: null, value_attribute: null, group_by: ['service_name'] }],
[
{
id: 'm2',
metric_name: 'sum_count',
filter_group: FILTER_GROUPS[4],
value_attribute: 'count',
group_by: ['severity_text', 'attributes.http.method', 'resource_attributes.k8s.pod.name'],
},
],
[
{
id: 'm3',
metric_name: 'hist_ms',
filter_group: null,
value_attribute: 'ms',
group_by: ['attributes.payload'],
},
],
[
{
id: 'm4',
metric_name: 'bad group',
filter_group: { type: 'AND', values: 'x' },
value_attribute: null,
group_by: 'notarray' as unknown as string[],
},
],
]
out.metric_tally = metricRuleSets.flatMap((rows) =>
[false, true].map((withDefaults) => {
let rules = compileMetricRules(rows)
if (withDefaults) {
rules = withDefaultMetricRules(rules, 'logs')
}
const tallies = createBatchTallies()
const batch = records.slice(0, 40)
tallyRecords(rules, batch, tallies, NOW_MS, () => true)
let overflow = 0
for (const count of tallies.seriesOverflow.values()) {
overflow += count
}
return {
input: {
rules: rows,
with_default_rules: withDefaults,
now_ms: NOW_MS,
records: batch.map(recordToJson),
},
output: {
payload: canonicalizeAvroValue(buildMetricRulesOtlpPayload(rules, tallies, NOW_MS)),
value_skipped: tallies.valueSkipped,
eval_errors: tallies.evalErrors,
series_overflow: overflow,
},
}
})
)
out.content_weights = [records.slice(0, 10), records.slice(10, 40), records.slice(40, 41)].map((batch) => ({
input: batch.map(recordToJson),
output: [...measureContentWeights(batch).values()],
}))
const key = 'k1'
const signedAt = new Date(NOW_MS - 2000)
const created = new Date(NOW_MS).toISOString()
const verifier = new InternalLogsVerifier(`${key},k0`, '1,2', {
hog_flow: { billed: false, teamRules: false, serviceName: 'posthog-internal-workflows' },
billed_src: { billed: true, teamRules: true, serviceName: 'billed-src' },
})
const signatures: [string | undefined, number, string | undefined][] = [
[undefined, 1, created],
[signInternalLogs(key, 'hog_flow', 1, signedAt), 1, created],
[signInternalLogs('k0', 'hog_flow', 1, signedAt), 1, created],
[signInternalLogs(key, 'hog_flow', 1, signedAt), 3, created],
[signInternalLogs(key, 'hog_flow', 2, signedAt), 1, created],
[signInternalLogs(key, 'billed_src', 2, signedAt), 2, created],
[signInternalLogs(key, 'constructor', 1, signedAt), 1, created],
[signInternalLogs(key, 'unknown', 1, signedAt), 1, created],
[signInternalLogs(key, 'hog_flow', 1, new Date(NOW_MS - 301_000)), 1, created],
[signInternalLogs(key, 'hog_flow', 1, new Date(NOW_MS - 299_000)), 1, created],
[signInternalLogs(key, 'hog_flow', 1, signedAt), 1, undefined],
[signInternalLogs(key, 'hog_flow', 1, signedAt), 1, 'Wed, 15 Jan 2025 12:37:13 GMT'],
['v1.hog_flow.1736944631.00', 1, created],
['v1.HOG.1736944631.00', 1, created],
['v1.hog_flow.0123456789.00', 1, created],
['garbage', 1, created],
]
// The Rust port has no producer, so the vectors leave out the producer-only `serviceName`.
out.internal_verify = signatures.map(([signature, teamId, createdAt]) => {
const { serviceName: _, ...output } = verifier.verify(signature, teamId, createdAt) ?? {}
return {
input: { signature: signature ?? null, team_id: teamId, created_at: createdAt ?? null },
output: Object.keys(output).length > 0 ? output : null,
}
})
return out
}
async function settleNormalize(cases: Record<string, Case[]>): Promise<void> {
for (const c of cases.normalize) {
const record = c.output as LogRecord
const settings = (c.input as { settings: Record<string, unknown> }).settings
const pii = await transformDecodedLogRecordsInPlace([record], settings)
c.output = { record: recordToJson(record), pii_replacements: pii.piiReplacements }
}
}
describe('logs ingestion module parity vectors', () => {
if (RECORD) {
it('records the vectors', async () => {
const cases = buildCases()
await settleNormalize(cases)
rmSync(DIR, { recursive: true, force: true })
mkdirSync(DIR, { recursive: true })
for (const [name, list] of Object.entries(cases)) {
writeFileSync(join(DIR, `${name}.json`), JSON.stringify({ cases: list }, null, 1) + '\n')
}
})
return
}
const files = existsSync(DIR) ? readdirSync(DIR).filter((f) => f.endsWith('.json')) : []
it('has vectors', () => {
expect(files.length).toBeGreaterThan(0)
})
it('matches the recorded vectors', async () => {
const cases = buildCases()
await settleNormalize(cases)
for (const file of files) {
const recorded = parseJSON(readFileSync(join(DIR, file), 'utf8')) as { cases: Case[] }
const actual = parseJSON(JSON.stringify({ cases: cases[file.replace(/\.json$/, '')] }))
expect({ file, ...actual }).toStrictEqual({ file, ...recorded })
}
})
})