:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { convertHogToJS } from '@posthog/hogvm'
import type { HogFunctionType } from '~/cdp/types'
import { sanitizeLogMessage } from '~/cdp/utils'
import { execHogImmediate } from '~/cdp/utils/hog-exec'
import { decodeLogAttributeValue, encodeLogAttributeValue } from '../attribute-value'
import type { LogRecord } from '../log-record-avro'
import { MAX_LOG_RECORD_BYTES, logRecordSizeBytes } from '../log-record-size'
import { idToHex } from '../metrics-rules/tally'
// Per-record execution primitives for log transformations. Pure functions, no I/O:
// orchestration (function fetching, budgets, monitoring) lives in the transformer service.
/** Hard per-record VM kill. ~100x the p99 of realistic scrub programs (see benchmarks/). */
export const DEFAULT_LOG_TRANSFORMATION_TIMEOUT_MS = 10
/** Log records are ~1KB; transformations have no business allocating anywhere near this. */
export const LOG_TRANSFORMATION_MEMORY_LIMIT_BYTES = 8 * 1024 * 1024
/** Max captured print() entries per invocation — far lower than destinations' 25,
* since a transformation can run hundreds of thousands of times per second. */
export const MAX_LOG_TRANSFORMATION_PRINT_LOGS = 5
/** Capture bounds whole ingest requests at 2MB; a transformation must not inflate a
* stored record past that boundary. The cap applies to the complete transformed
* output (body + severity + attribute map totals); oversize output is invalid. */
export const MAX_TRANSFORMED_FIELD_BYTES = MAX_LOG_RECORD_BYTES
export interface LogTransformationGlobals {
project: { id: number; name: string; url: string }
record: {
body: string | null
attributes: Record<string, string>
resource_attributes: Record<string, string>
severity_text: string | null
severity_number: number | null
service_name: string | null
instrumentation_scope: string | null
event_name: string | null
timestamp: number | null
observed_timestamp: number | null
trace_id: string | null
span_id: string | null
}
inputs: Record<string, unknown>
}
export type LogTransformationOutcome =
| { status: 'mutated'; durationMs: number; logs: string[] }
| { status: 'dropped'; durationMs: number; logs: string[] }
| { status: 'failed'; durationMs: number; logs: string[]; error: string }
export function buildLogRecordGlobals(
record: LogRecord,
project: LogTransformationGlobals['project'],
inputs: Record<string, unknown>
): LogTransformationGlobals {
return {
project,
record: buildLogRecordGlobalsRecord(record),
inputs,
}
}
/** The `record` half of the globals, with attribute maps decoded once. */
export function buildLogRecordGlobalsRecord(record: LogRecord): LogTransformationGlobals['record'] {
return {
body: record.body ?? null,
attributes: decodeAttributeMap(record.attributes),
resource_attributes: decodeAttributeMap(record.resource_attributes),
severity_text: record.severity_text ?? null,
severity_number: record.severity_number ?? null,
service_name: record.service_name ?? null,
instrumentation_scope: record.instrumentation_scope ?? null,
event_name: record.event_name ?? null,
timestamp: record.timestamp ?? null,
observed_timestamp: record.observed_timestamp ?? null,
trace_id: idToHex(record.trace_id, 16),
span_id: idToHex(record.span_id, 8),
}
}
/**
* Refreshes the mutable record fields of an already-built globals object from the
* (possibly transformed) record. Lets the per-function loop reuse one globals build
* per record instead of re-decoding every attribute map per function.
*/
export function refreshLogRecordGlobalsRecord(target: LogTransformationGlobals['record'], record: LogRecord): void {
target.body = record.body ?? null
target.attributes = decodeAttributeMap(record.attributes)
target.resource_attributes = decodeAttributeMap(record.resource_attributes)
target.severity_text = record.severity_text ?? null
target.severity_number = record.severity_number ?? null
target.service_name = record.service_name ?? null
target.instrumentation_scope = record.instrumentation_scope ?? null
target.event_name = record.event_name ?? null
target.timestamp = record.timestamp ?? null
target.observed_timestamp = record.observed_timestamp ?? null
target.trace_id = idToHex(record.trace_id, 16)
target.span_id = idToHex(record.span_id, 8)
}
function decodeAttributeMap(map: Record<string, string> | null | undefined): Record<string, string> {
const out: Record<string, string> = {}
for (const [key, value] of Object.entries(map ?? {})) {
out[key] = decodeLogAttributeValue(value)
}
return out
}
/**
* Encodes a decoded attribute map back to the wire form. Values the transformation left
* untouched keep their exact original wire encoding — the decode/encode pair is lossy for
* string values that look like JSON scalars ('"3"' decodes to '3', which re-encodes as the
* number 3), so round-tripping through the original is the only faithful option.
*/
function encodeAttributeMap(
map: Record<string, string>,
original: Record<string, string> | null | undefined,
// The map already decoded by the globals build: comparing against it decides
// "untouched" without decoding every original wire value a second time.
originalDecoded?: Record<string, string>
): Record<string, string> {
const out: Record<string, string> = {}
for (const [key, value] of Object.entries(map)) {
const originalValue = original?.[key]
if (originalValue !== undefined) {
const decoded = originalDecoded?.[key] ?? decodeLogAttributeValue(originalValue)
if (decoded === value) {
out[key] = originalValue
continue
}
}
out[key] = encodeLogAttributeValue(value)
}
return out
}
function coerceStringMap(value: unknown): Record<string, string> | null {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
return null
}
const result: Record<string, string> = {}
for (const [key, entry] of Object.entries(value as Record<string, unknown>)) {
if (entry === null || entry === undefined) {
continue
}
if (typeof entry === 'string') {
result[key] = entry
} else {
try {
result[key] = JSON.stringify(entry)
} catch {
// Cyclic values can't be encoded; treat the whole result as invalid so
// the failure stays inside the executor's normal accounting.
return null
}
}
if (Buffer.byteLength(result[key]) > MAX_TRANSFORMED_FIELD_BYTES) {
return null
}
}
return result
}
/**
* Merges a transformation's return value back into the record, in place.
*
* Only `body`, `attributes`, `severity_text`, and `resource_attributes` are writable.
* Timestamps, trace/span ids, service identity, and `bytes_uncompressed` are read-only:
* billing bytes are computed at capture time, and scrubbing must never change what a
* customer is billed.
*
* Returns 'invalid' (record untouched) when the result is not record-shaped — the
* caller treats that as a failure and fails open.
*/
function redactString(value: string, sensitiveValues: string[]): string {
let out = value
for (const sensitive of sensitiveValues) {
out = out.replaceAll(sensitive, '***REDACTED***')
}
return out
}
function redactSensitiveStrings(
value: unknown,
sensitiveValues: string[] | undefined,
// Memo doubles as cycle guard: a transformation can return a cyclic record, and
// an unbounded recursion here would throw past the caller's failure accounting.
seen: WeakMap<object, unknown> = new WeakMap()
): unknown {
if (!sensitiveValues?.length) {
return value
}
if (typeof value === 'string') {
return redactString(value, sensitiveValues)
}
if (Array.isArray(value)) {
const existing = seen.get(value)
if (existing) {
return existing
}
const out: unknown[] = []
seen.set(value, out)
for (const item of value) {
out.push(redactSensitiveStrings(item, sensitiveValues, seen))
}
return out
}
if (value && typeof value === 'object') {
const existing = seen.get(value)
if (existing) {
return existing
}
const out: Record<string, unknown> = {}
seen.set(value, out)
for (const [key, item] of Object.entries(value)) {
// Keys need redacting as much as values do: Hog can assign a dynamic key
// (`rec.attributes[inputs.apiKey] := …`), so redacting only values leaves the
// same exfiltration path open one level up. Two keys collapsing onto the same
// redacted name is fine — both sides of that collision hold the secret.
out[redactString(key, sensitiveValues)] = redactSensitiveStrings(item, sensitiveValues, seen)
}
return out
}
return value
}
export function applyTransformResult(
record: LogRecord,
execResult: unknown,
decodedOriginals?: {
attributes?: Record<string, string>
resource_attributes?: Record<string, string>
}
): 'mutated' | 'dropped' | 'invalid' {
if (execResult === null || execResult === undefined || execResult === false) {
return 'dropped'
}
if (typeof execResult !== 'object' || Array.isArray(execResult)) {
return 'invalid'
}
const result = execResult as Record<string, unknown>
// Validate everything before mutating anything, so an invalid result leaves the
// record fully untouched rather than half-applied.
let body: string | null | undefined = undefined
if ('body' in result) {
if (result.body !== null && typeof result.body !== 'string') {
return 'invalid'
}
body = result.body
}
let severityText: string | null | undefined = undefined
if ('severity_text' in result) {
if (result.severity_text !== null && typeof result.severity_text !== 'string') {
return 'invalid'
}
severityText = result.severity_text
}
// Like `body`, an explicit null clears the field; an absent key leaves it untouched.
let attributes: Record<string, string> | null | undefined = undefined
if ('attributes' in result) {
if (result.attributes === null) {
attributes = null
} else {
attributes = coerceStringMap(result.attributes)
if (attributes === null) {
return 'invalid'
}
}
}
let resourceAttributes: Record<string, string> | null | undefined = undefined
if ('resource_attributes' in result) {
if (result.resource_attributes === null) {
resourceAttributes = null
} else {
resourceAttributes = coerceStringMap(result.resource_attributes)
if (resourceAttributes === null) {
return 'invalid'
}
}
}
// Cap the complete transformed output — body, severity, and attribute map
// totals together — so no combination of writable fields can inflate a
// stored record past the boundary.
// Retained fields count too: a partial result (e.g. {body}) must not slip the
// record past the cap by riding on large fields it left untouched.
const finalBody = body !== undefined ? body : record.body
const finalSeverity = severityText !== undefined ? severityText : record.severity_text
const finalAttributes = attributes !== undefined ? attributes : record.attributes
const finalResourceAttributes = resourceAttributes !== undefined ? resourceAttributes : record.resource_attributes
const totalBytes = logRecordSizeBytes({
body: finalBody,
severity_text: finalSeverity,
attributes: finalAttributes,
resource_attributes: finalResourceAttributes,
})
if (totalBytes > MAX_TRANSFORMED_FIELD_BYTES) {
return 'invalid'
}
if (body !== undefined) {
record.body = body
}
if (severityText !== undefined) {
record.severity_text = severityText
}
if (attributes !== undefined) {
record.attributes =
attributes === null ? null : encodeAttributeMap(attributes, record.attributes, decodedOriginals?.attributes)
}
if (resourceAttributes !== undefined) {
record.resource_attributes =
resourceAttributes === null
? null
: encodeAttributeMap(
resourceAttributes,
record.resource_attributes,
decodedOriginals?.resource_attributes
)
}
return 'mutated'
}
/**
* Resolves a function's input values against log globals. Inputs may reference earlier
* inputs, so resolution happens in declared order; hog-templated inputs execute their
* compiled bytecode with the accumulated `inputs` visible.
*
* Throws on the first unresolvable input — callers decide the failure policy
* (the pipeline fails open per record; the test-invocation API surfaces the error).
*/
export function resolveLogTransformationInputs(
fn: Pick<HogFunctionType, 'inputs' | 'encrypted_inputs'>,
globals: Omit<LogTransformationGlobals, 'inputs'>,
timeoutMs: number
): { inputs: Record<string, unknown>; durationMs: number } {
const inputs: Record<string, unknown> = {}
const allInputs = { ...fn.inputs, ...fn.encrypted_inputs }
// VM time spent on templates counts toward the same budgets as the code body —
// record-referencing templates run per record and would otherwise bypass them.
let durationMs = 0
const entries = Object.entries(allInputs).sort(([, a], [, b]) => (a?.order ?? -1) - (b?.order ?? -1))
for (const [key, input] of entries) {
if (input?.bytecode && (input.templating ?? 'hog') === 'hog') {
const {
execResult,
error,
durationMs: execMs,
} = execHogImmediate(input.bytecode, {
globals: { ...globals, inputs },
timeout: timeoutMs,
maxAsyncSteps: 0,
memoryLimit: LOG_TRANSFORMATION_MEMORY_LIMIT_BYTES,
})
durationMs += execMs
if (error || execResult?.error || !execResult?.finished) {
throw new Error(`Could not resolve input '${key}': ${error ?? execResult?.error}`)
}
// Hog objects surface as Maps; convert like the body's result so inputs
// hold plain JS values for globals, encoders, and redaction.
inputs[key] = convertHogToJS(execResult.result)
} else {
inputs[key] = input?.value
}
}
return { inputs, durationMs }
}
export interface ExecuteLogTransformationOptions {
timeoutMs?: number
/** Values redacted from captured print() output (e.g. secret input values). */
sensitiveValues?: string[]
}
/**
* Runs one compiled transformation against one record's globals and merges the result
* into the record in place. Synchronous and CPU-only: no async functions are registered,
* so customer code cannot perform I/O.
*/
export function executeLogTransformation(
bytecode: unknown,
record: LogRecord,
globals: LogTransformationGlobals,
options: ExecuteLogTransformationOptions = {}
): LogTransformationOutcome {
const timeoutMs = options.timeoutMs ?? DEFAULT_LOG_TRANSFORMATION_TIMEOUT_MS
const logs: string[] = []
const { execResult, error, durationMs } = execHogImmediate(bytecode, {
globals,
timeout: timeoutMs,
maxAsyncSteps: 0,
memoryLimit: LOG_TRANSFORMATION_MEMORY_LIMIT_BYTES,
functions: {
print: (...args: unknown[]) => {
if (logs.length < MAX_LOG_TRANSFORMATION_PRINT_LOGS) {
logs.push(sanitizeLogMessage(args, options.sensitiveValues))
}
},
},
})
if (error || execResult?.error) {
return { status: 'failed', durationMs, logs, error: String(error ?? execResult?.error) }
}
if (!execResult?.finished) {
// With no async functions registered the VM always runs to completion or throws;
// a paused VM here means a bug, not customer error. Treat as failure (fail-open).
return { status: 'failed', durationMs, logs, error: 'VM did not finish execution' }
}
// Redact secrets from the returned record before it's applied: a transformation
// can copy a decrypted input into a writable field, and Logs readers must not be
// able to recover encrypted input values that way.
const converted = redactSensitiveStrings(convertHogToJS(execResult.result), options.sensitiveValues)
const applied = applyTransformResult(record, converted, {
attributes: globals.record.attributes,
resource_attributes: globals.record.resource_attributes,
})
if (applied === 'invalid') {
return {
status: 'failed',
durationMs,
logs,
error: 'Transformation must return the record (optionally mutated) or null to drop it',
}
}
if (applied === 'dropped') {
return { status: 'dropped', durationMs, logs }
}
return { status: 'mutated', durationMs, logs }
}