:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
//! The flatten keeps or evicts attributes by JavaScript key order: array-index keys first, then insertion order.
//! Only the Prometheus counter counts enrichment skips, where the TypeScript also adds an OpenTelemetry counter.
use std::collections::BTreeMap;
use std::sync::Arc;
use hashbrown::HashTable;
use metrics::counter;
use serde::Deserialize;
use crate::avro::{AvroMap, AvroValue};
use crate::body_parse::{parse_log_body_for_ingestion, LogBodyParseResult};
use crate::js::object::hash_key;
use crate::js::{
is_array_index, json_parse, json_stringify, json_stringify_into, quote_js_string,
CharOrSurrogate, JsKey, JsObject, JsString, JsValue,
};
use crate::pii::{scrub_log_record, PiiScrubStats, EMPTY_PII};
use crate::record::LogRecord;
pub const MAX_LOG_RECORD_BYTES: usize = 1024 * 1024;
pub const MAX_JSON_ATTRIBUTES: usize = 50;
pub const MAX_JSON_NODES: usize = 10_000;
pub const MAX_JSON_DEPTH: usize = 128;
// The flatten must stop before it can reach a value that `json_parse` did not build.
const _: () = assert!(MAX_JSON_DEPTH < crate::js::MAX_BUILT_DEPTH);
/// A missing key reads as `undefined`.
#[derive(Debug, Clone, Default, PartialEq, Deserialize)]
pub struct LogsSettings {
#[serde(default)]
pub capture_console_logs: Option<bool>,
#[serde(default)]
pub json_parse_logs: Option<bool>,
#[serde(default)]
pub json_parse_logs_attribute_key: Option<String>,
#[serde(default)]
pub pii_scrub_logs: Option<bool>,
#[serde(default)]
pub retention_days: Option<f64>,
#[serde(default)]
pub retention_last_updated: Option<String>,
}
impl LogsSettings {
/// An empty key is off, as `Boolean(key)` is.
pub fn attribute_key(&self) -> Option<&str> {
self.json_parse_logs_attribute_key
.as_deref()
.filter(|k| !k.is_empty())
}
}
pub fn log_record_size_bytes(record: &LogRecord) -> usize {
let mut bytes = 0;
for value in [record.body(), record.severity_text()]
.into_iter()
.flatten()
{
bytes += value.len();
}
for attributes in [record.attributes(), record.resource_attributes()]
.into_iter()
.flatten()
{
for (key, value) in attributes.iter_strings() {
bytes += key.len() + value.len();
}
}
bytes
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum JsonEnrichmentSource {
Body,
SelectedAttribute,
}
impl JsonEnrichmentSource {
fn as_str(self) -> &'static str {
match self {
JsonEnrichmentSource::Body => "body",
JsonEnrichmentSource::SelectedAttribute => "selected_attribute",
}
}
}
fn record_json_enrichment_skip(reason: &'static str, source: JsonEnrichmentSource) {
counter!(
"logs_ingestion_json_enrichment_skipped_total",
"reason" => reason,
"source" => source.as_str()
)
.increment(1);
}
/// Every character the regex consumes is ASCII, so 64 UTF-16 units are 64 bytes up to where the scan stops.
fn looks_like_json(value: &str) -> bool {
let bytes = &value.as_bytes()[..value.len().min(64)];
let is_space = |b: u8| matches!(b, b' ' | b'\t' | b'\r' | b'\n');
let mut i = 0;
while i < bytes.len() && is_space(bytes[i]) {
i += 1;
}
if i < bytes.len() && bytes[i] == b'"' {
i += 1;
}
loop {
match bytes.get(i) {
Some(&b) if is_space(b) => i += 1,
Some(b'\\') if matches!(bytes.get(i + 1), Some(b'n' | b'r' | b't')) => i += 2,
Some(b'{' | b'[') => return true,
_ => return false,
}
}
}
pub fn sniff_json_log_attributes<'a>(
records: impl IntoIterator<Item = &'a LogRecord>,
attribute_key: &str,
team_id: i64,
) {
let team_id: Arc<str> = Arc::from(team_id.to_string());
for record in records {
let outcome = match record.attributes().and_then(|a| a.get(attribute_key)) {
None => "missing_key",
Some(value) => match value.as_str() {
Some(text) if looks_like_json(text) => "looks_like_json",
_ => "other",
},
};
counter!(
"logs_ingestion_json_attribute_sniff_total",
"team_id" => team_id.clone(),
"outcome" => outcome
)
.increment(1);
}
}
fn count_json_nodes_within_budget(value: &JsValue, depth: usize, budget: usize) -> Option<usize> {
let mut stack = vec![(value, depth)];
let mut nodes = 1usize;
while let Some((value, depth)) = stack.pop() {
if nodes > budget || depth > MAX_JSON_DEPTH {
return None;
}
let children: Box<dyn Iterator<Item = &JsValue>> = match value {
JsValue::Array(items) => Box::new(items.iter()),
JsValue::Object(object) => Box::new(object.iter().map(|(_, v)| v)),
_ => continue,
};
for child in children {
nodes += 1;
if nodes > budget {
return None;
}
stack.push((child, depth + 1));
}
}
Some(nodes)
}
pub type FlatAttributes = Vec<(JsString, String)>;
#[derive(Debug, Clone, PartialEq)]
pub struct FlattenResult {
pub values: JsObject,
pub attributes: FlatAttributes,
}
/// `result` and `attributes` always hold the same keys, so one entry holds both.
struct FlatEntry {
key: JsString,
value: Option<JsValue>,
serialized: String,
}
#[derive(Clone, Copy)]
enum FlatPos {
Index(u32),
Named(Option<usize>),
}
#[derive(Default)]
struct FlatObject {
indexed: BTreeMap<u32, FlatEntry>,
named: Vec<FlatEntry>,
named_position: HashTable<usize>,
}
impl FlatObject {
fn named_index(&self, key: &JsString) -> Option<usize> {
self.named_position
.find(hash_key(key), |&i| self.named[i].key == *key)
.copied()
}
fn get(&self, key: &JsString) -> Option<&FlatEntry> {
self.at(self.locate(key))
}
fn locate(&self, key: &JsString) -> FlatPos {
match is_index_key(key) {
Some(n) => FlatPos::Index(n),
None => FlatPos::Named(self.named_index(key)),
}
}
fn at(&self, pos: FlatPos) -> Option<&FlatEntry> {
match pos {
FlatPos::Index(n) => self.indexed.get(&n),
FlatPos::Named(i) => i.map(|i| &self.named[i]),
}
}
/// Stores `entry` at `pos`, which `locate` gave for its key with no removal since.
fn set_at(&mut self, pos: FlatPos, entry: FlatEntry) {
match pos {
FlatPos::Index(n) => {
self.indexed.insert(n, entry);
}
FlatPos::Named(Some(i)) => self.named[i] = entry,
FlatPos::Named(None) => {
let named = &self.named;
self.named_position
.insert_unique(hash_key(&entry.key), named.len(), |&j| {
hash_key(&named[j].key)
});
self.named.push(entry);
}
}
}
fn remove(&mut self, key: &JsString) {
if let Some(n) = is_index_key(key) {
self.indexed.remove(&n);
} else if let Some(i) = self.named_index(key) {
self.named.remove(i);
self.named_position.clear();
let named = &self.named;
for (j, entry) in named.iter().enumerate() {
self.named_position
.insert_unique(hash_key(&entry.key), j, |&k| hash_key(&named[k].key));
}
}
}
fn last_key(&self) -> Option<&JsString> {
match self.named.last() {
Some(entry) => Some(&entry.key),
None => self.indexed.values().next_back().map(|entry| &entry.key),
}
}
fn into_entries(self) -> impl Iterator<Item = FlatEntry> {
self.indexed.into_values().chain(self.named)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ArrayMode {
Indexed,
String,
}
fn is_index_key(key: &JsString) -> Option<u32> {
key.as_str().and_then(is_array_index)
}
enum Leaf<'a> {
Value(&'a JsValue),
Stringified(String),
}
/// Recurses in the TypeScript stack order with the same budget updates per node, so it evicts exactly where the TypeScript does.
struct Flattener {
max_attributes: usize,
max_bytes: usize,
array_mode: ArrayMode,
keep_values: bool,
nodes: usize,
path_bytes: usize,
attribute_bytes: i64,
attribute_count: usize,
last_retained_key: JsString,
result: FlatObject,
}
impl Flattener {
/// `None` means a budget ran out.
fn visit(&mut self, node: &JsValue, path: &mut JsString, depth: usize) -> Option<()> {
if depth > MAX_JSON_DEPTH || self.path_bytes > self.max_bytes {
return None;
}
match node {
JsValue::Array(_) if self.array_mode == ArrayMode::String => {
let array_nodes =
count_json_nodes_within_budget(node, depth, MAX_JSON_NODES - self.nodes + 1)?;
self.nodes += array_nodes - 1;
self.leaf(path, Leaf::Stringified(json_stringify(node)))
}
JsValue::Array(items) => {
if items.len() + self.nodes > MAX_JSON_NODES {
return None;
}
self.nodes += items.len();
let keys = (0..items.len()).map(|i| JsKey::Index(i as u32));
self.add_child_paths(path, keys)?;
self.visit_children(
path,
depth,
items
.iter()
.enumerate()
.map(|(i, v)| (JsKey::Index(i as u32), v)),
)
}
JsValue::Object(object) => {
let n = object.len();
if n > 0 && self.nodes + n > MAX_JSON_NODES {
return None;
}
self.nodes += n;
self.add_child_paths(path, object.entries().map(|(k, _)| k))?;
self.visit_children(path, depth, object.entries())
}
_ => self.leaf(path, Leaf::Value(node)),
}
}
/// Adds the byte length of every child path, as the TypeScript does before it descends.
fn add_child_paths<'k>(
&mut self,
path: &JsString,
keys: impl Iterator<Item = JsKey<'k>>,
) -> Option<()> {
let base = path.utf8_byte_len();
for key in keys {
self.path_bytes += if base == 0 {
key.utf8_byte_len()
} else {
base + 1 + key.utf8_byte_len()
};
if self.path_bytes > self.max_bytes {
return None;
}
}
Some(())
}
fn visit_children<'v>(
&mut self,
path: &mut JsString,
depth: usize,
children: impl Iterator<Item = (JsKey<'v>, &'v JsValue)>,
) -> Option<()> {
let base = path.utf8_byte_len();
let base_surrogates = path.lone_surrogate_count();
for (key, child) in children {
if base > 0 {
path.push_str(".");
}
key.push_onto(path);
self.visit(child, path, depth + 1)?;
path.truncate_to(base, base_surrogates);
}
Some(())
}
fn leaf(&mut self, key: &JsString, leaf: Leaf<'_>) -> Option<()> {
if key.is_empty() || *key == "__proto__" || self.max_attributes == 0 {
return Some(());
}
let pos = self.result.locate(key);
let exists = self.result.at(pos).is_some();
if !exists && self.attribute_count >= self.max_attributes {
let Some(index) = is_index_key(key) else {
return Some(());
};
let last_key = self.last_retained_key.clone();
if let Some(last_index) = is_index_key(&last_key) {
if last_index <= index {
return Some(());
}
}
if let Some(entry) = self.result.get(&last_key) {
self.attribute_bytes -= (last_key.utf8_byte_len() + entry.serialized.len()) as i64;
}
self.result.remove(&last_key);
self.attribute_count -= 1;
}
let (serialized, value) = match leaf {
Leaf::Value(JsValue::Null) => (
"\"null\"".to_string(),
self.keep_values.then(|| JsValue::string("null")),
),
Leaf::Value(v) => {
let mut serialized = String::new();
if let JsValue::String(s) = v {
serialized.reserve(s.utf8_byte_len() + 2);
}
json_stringify_into(v, &mut serialized);
(serialized, self.keep_values.then(|| v.clone()))
}
Leaf::Stringified(text) => {
let text = JsString::from(text);
let mut serialized = String::with_capacity(text.utf8_byte_len() + 2);
quote_js_string(&text, &mut serialized);
(serialized, self.keep_values.then(|| JsValue::String(text)))
}
};
let value_bytes = serialized.len() as i64;
// An eviction above removes another key only when this key is absent, so `pos` holds.
self.attribute_bytes += match self.result.at(pos) {
Some(previous) => value_bytes - previous.serialized.len() as i64,
None => (key.utf8_byte_len() as i64) + value_bytes,
};
if self.attribute_bytes > self.max_bytes as i64 {
return None;
}
self.result.set_at(
pos,
FlatEntry {
key: key.clone(),
value,
serialized,
},
);
if !exists {
self.attribute_count += 1;
if self.attribute_count == self.max_attributes {
self.last_retained_key = self.result.last_key().cloned().unwrap_or_default();
}
}
Some(())
}
}
fn flatten_entries(
obj: &JsValue,
prefix: &JsString,
max_attributes: usize,
max_bytes: usize,
array_mode: ArrayMode,
keep_values: bool,
) -> Option<FlatObject> {
let mut flattener = Flattener {
max_attributes,
max_bytes,
array_mode,
keep_values,
nodes: 1,
path_bytes: prefix.utf8_byte_len(),
attribute_bytes: 0,
attribute_count: 0,
last_retained_key: JsString::new(),
result: FlatObject::default(),
};
let mut path = prefix.clone();
flattener.visit(obj, &mut path, 0)?;
Some(flattener.result)
}
pub fn flatten_json_with_budget(
obj: &JsValue,
prefix: &JsString,
max_attributes: usize,
max_bytes: usize,
array_mode: ArrayMode,
) -> Option<FlattenResult> {
let result = flatten_entries(obj, prefix, max_attributes, max_bytes, array_mode, true)?;
let mut values = JsObject::new();
let mut attributes = Vec::new();
for entry in result.into_entries() {
values.insert(entry.key.clone(), entry.value.expect("values are kept"));
attributes.push((entry.key, entry.serialized));
}
Some(FlattenResult { values, attributes })
}
fn flatten_attributes(
obj: &JsValue,
prefix: &JsString,
array_mode: ArrayMode,
) -> Option<FlatAttributes> {
let result = flatten_entries(
obj,
prefix,
MAX_JSON_ATTRIBUTES,
MAX_LOG_RECORD_BYTES,
array_mode,
false,
)?;
Some(
result
.into_entries()
.map(|entry| (entry.key, entry.serialized))
.collect(),
)
}
pub fn flatten_json(obj: &JsValue) -> Option<JsObject> {
flatten_json_with_budget(
obj,
&JsString::new(),
MAX_JSON_ATTRIBUTES,
MAX_LOG_RECORD_BYTES,
ArrayMode::Indexed,
)
.map(|r| r.values)
}
fn json_attributes_from_body_parse(body_parse: &LogBodyParseResult) -> FlatAttributes {
let LogBodyParseResult::JsonObjectOrArray { value } = body_parse else {
return Vec::new();
};
match flatten_attributes(value, &JsString::new(), ArrayMode::Indexed) {
Some(attributes) => attributes,
None => {
record_json_enrichment_skip("flatten_budget", JsonEnrichmentSource::Body);
Vec::new()
}
}
}
pub fn extract_json_attributes_from_body(body: Option<&str>) -> FlatAttributes {
if let Some(body) = body {
if body.len() > MAX_LOG_RECORD_BYTES {
record_json_enrichment_skip("input_size", JsonEnrichmentSource::Body);
return Vec::new();
}
}
json_attributes_from_body_parse(&parse_log_body_for_ingestion(body))
}
/// A key with a lone surrogate merges here with a U+FFFD key, where JavaScript keeps two keys. See PORTING.md.
fn add_json_attributes(
record: &mut LogRecord,
json_attributes: FlatAttributes,
mut record_bytes: usize,
source: JsonEnrichmentSource,
) {
if json_attributes.is_empty() {
return;
}
let existing = record.attributes();
for (key, value) in &json_attributes {
let present = match (existing, key.as_str()) {
(Some(attributes), Some(key)) => attributes.contains_key(key),
_ => false,
};
if !present {
record_bytes += key.utf8_byte_len() + value.len();
if record_bytes > MAX_LOG_RECORD_BYTES {
record_json_enrichment_skip("output_size", source);
return;
}
}
}
let mut merged = AvroMap::new();
for (key, value) in json_attributes {
merged.define(key.into_string_lossy(), AvroValue::String(value));
}
if let Some(AvroValue::Map(attributes)) = record.get_mut("attributes") {
for (key, value) in std::mem::take(attributes).into_entries() {
merged.define(key, value);
}
}
record.set("attributes", AvroValue::Map(merged));
}
/// Pass `body_parse` to reuse a parse of the same body.
pub fn enrich_log_record_with_json_attributes(
record: &mut LogRecord,
body_parse: Option<&LogBodyParseResult>,
) {
let Some(body) = record.body().filter(|b| !b.is_empty()) else {
return;
};
if body.len() > MAX_LOG_RECORD_BYTES {
record_json_enrichment_skip("input_size", JsonEnrichmentSource::Body);
return;
}
let owned;
let parse = match body_parse {
Some(parse) => parse,
None => {
owned = parse_log_body_for_ingestion(Some(body));
&owned
}
};
if !matches!(parse, LogBodyParseResult::JsonObjectOrArray { .. }) {
return;
}
let record_bytes = log_record_size_bytes(record);
if record_bytes > MAX_LOG_RECORD_BYTES {
record_json_enrichment_skip("input_size", JsonEnrichmentSource::Body);
return;
}
let json_attributes = json_attributes_from_body_parse(parse);
add_json_attributes(
record,
json_attributes,
record_bytes,
JsonEnrichmentSource::Body,
);
}
/// A raw lone surrogate inside a string literal means its `\uXXXX` escape, so the text is rewritten with escapes.
fn json_parse_js(text: &JsString) -> Option<JsValue> {
if let Some(text) = text.as_str() {
return json_parse(text).ok();
}
let mut rewritten = String::new();
let mut backslashes = 0usize;
for unit in text.chars_or_surrogates() {
match unit {
CharOrSurrogate::Char(c) => {
backslashes = if c == '\\' { backslashes + 1 } else { 0 };
rewritten.push(c);
}
CharOrSurrogate::Surrogate(s) => {
if backslashes % 2 == 1 {
return None;
}
backslashes = 0;
rewritten.push_str(&format!("\\u{s:04x}"));
}
}
}
json_parse(&rewritten).ok()
}
pub fn enrich_log_record_from_json_attribute(record: &mut LogRecord, key: &str) {
let Some(raw) = record.attributes().and_then(|a| a.get(key)) else {
return;
};
let Some(raw) = raw.as_str() else {
return;
};
let record_bytes = log_record_size_bytes(record);
if record_bytes > MAX_LOG_RECORD_BYTES {
record_json_enrichment_skip("input_size", JsonEnrichmentSource::SelectedAttribute);
return;
}
let mut parsed = match json_parse(raw) {
Ok(parsed) => parsed,
Err(_) => return,
};
if let JsValue::String(inner) = &parsed {
parsed = match json_parse_js(inner) {
Some(parsed) => parsed,
None => return,
};
}
if !matches!(parsed, JsValue::Object(_)) {
return;
}
match flatten_attributes(&parsed, &JsString::from(key), ArrayMode::String) {
Some(attributes) => add_json_attributes(
record,
attributes,
record_bytes,
JsonEnrichmentSource::SelectedAttribute,
),
None => {
record_json_enrichment_skip("flatten_budget", JsonEnrichmentSource::SelectedAttribute)
}
}
}
/// Sender attributes win over selected-attribute fields, and those win over body fields.
pub fn transform_decoded_log_records_in_place(
records: &mut [LogRecord],
settings: &LogsSettings,
) -> PiiScrubStats {
let mut pii = EMPTY_PII;
if settings.pii_scrub_logs.unwrap_or(false) {
for record in records.iter_mut() {
pii.pii_replacements += scrub_log_record(record).pii_replacements;
}
}
if let Some(key) = settings.attribute_key() {
for record in records.iter_mut() {
enrich_log_record_from_json_attribute(record, key);
}
}
if settings.json_parse_logs.unwrap_or(false) {
// No enrichment changes a body, so one parse per record in this loop equals Node's parse-all-first order.
for record in records.iter_mut() {
let parse = parse_log_body_for_ingestion(
record.body().filter(|b| b.len() <= MAX_LOG_RECORD_BYTES),
);
enrich_log_record_with_json_attributes(record, Some(&parse));
}
}
pii
}
/// Cheapest first.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum BufferProcessingMode {
Passthrough,
DecodeOnly,
DecodeAndReencode,
}
pub fn buffer_processing_mode(
settings: &LogsSettings,
stage_count: usize,
has_visitor: bool,
) -> BufferProcessingMode {
let normalize_active = settings.json_parse_logs.unwrap_or(false)
|| settings.pii_scrub_logs.unwrap_or(false)
|| settings.attribute_key().is_some();
if normalize_active || stage_count > 0 {
BufferProcessingMode::DecodeAndReencode
} else if has_visitor {
BufferProcessingMode::DecodeOnly
} else {
BufferProcessingMode::Passthrough
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn sniff_matches_the_regex() {
for (value, expected) in [
("{", true),
(" \"{", true),
("\"\\n\\t [", true),
("\\n{", true),
("\"\"{", false),
("x{", false),
("\u{000B}{", false),
("\\x{", false),
("", false),
] {
assert_eq!(looks_like_json(value), expected, "{value:?}");
}
let far = format!("{}{{", " ".repeat(64));
assert!(!looks_like_json(&far));
}
#[test]
fn a_raw_lone_surrogate_parses_as_its_escape() {
let mut text = JsString::from("{\"a\":\"");
text.push_code_unit(0xD800);
text.push_str("\"}");
let JsValue::Object(o) = json_parse_js(&text).unwrap() else {
panic!("not an object");
};
assert_eq!(json_stringify(o.get("a").unwrap()), "\"\\ud800\"");
let mut escaped = JsString::from("\"\\");
escaped.push_code_unit(0xD800);
escaped.push_str("\"");
assert!(json_parse_js(&escaped).is_none());
}
}