Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

20.2 KB
Raw Blame
//! Routes each parsed rrweb event to the right scrubber by type/source, and walks a whole parsed
//! message (`{ windowId: Event[] }`).

use anyhow::{bail, Context, Result};
use simd_json::borrowed::{Object, Value};
use simd_json::prelude::Writable;
use simd_json::StaticNode;

use crate::allow_lists::AllowLists;
use crate::canvas::scrub_canvas_mutation;
use crate::context::Ctx;
use crate::css::{scrub_adopted_style_sheet, scrub_style_declaration, scrub_style_sheet_rule};
use crate::cv::{scrub_compressed_full_snapshot, scrub_compressed_mutation};
use crate::dom::{scrub_full_snapshot, scrub_mutation};
use crate::json::{
    as_array_mut, as_object, as_object_mut, as_small_uint, as_str, key, parse_untrusted,
    reject_if_too_deep, string_value,
};
use crate::json_ld::scrub_json_ld_payload;
use crate::text::scrub_text;
use crate::url::scrub_url;
use crate::value::{scrub_console_plugin, scrub_generic_field, scrub_network_plugin};

// RRWebEventType
pub const TYPE_FULL_SNAPSHOT: u8 = 2;
pub const TYPE_INCREMENTAL: u8 = 3;
pub const TYPE_META: u8 = 4;
pub const TYPE_CUSTOM: u8 = 5;
pub const TYPE_PLUGIN: u8 = 6;

// RRWebEventSource (incremental)
pub const SOURCE_MUTATION: u8 = 0;
pub const SOURCE_INPUT: u8 = 5;
pub const SOURCE_STYLESHEET_RULE: u8 = 8;
pub const SOURCE_CANVAS_MUTATION: u8 = 9;
pub const SOURCE_STYLE_DECLARATION: u8 = 13;
pub const SOURCE_ADOPTED_STYLESHEET: u8 = 15;

pub const NETWORK_PLUGIN: &str = "rrweb/network@1";
pub const CONSOLE_PLUGIN: &str = "rrweb/console@1";
pub(crate) const JSON_LD_EVENT_TAG: &str = "$json_ld";

/// Panic backstop for the anyhow-based entry points (see [`crate::unwind`]).
fn contain_panics<T>(f: impl FnOnce() -> Result<T>) -> Result<T> {
    crate::unwind::contain_unwind(f, |msg| anyhow::anyhow!("panic while anonymizing: {msg}"))
}

/// Anonymizes every event in a parsed message (`{ windowId: Event[] }`) in place. `Ok(None)` means
/// nothing changed (the caller can keep its original). `Err` means an event could not be anonymized —
/// fail closed, the caller must drop the message. A single `Ctx` spans the whole message so its image
/// memo is shared across every event (an image recurring across events is decoded once).
pub fn anonymize_message(allow: &AllowLists, json: &mut [u8]) -> Result<Option<String>> {
    contain_panics(|| {
        reject_if_too_deep(json, "eventsByWindowId")?;
        let mut root = parse_untrusted(json).context("parse eventsByWindowId json")?;
        let Some(obj) = as_object_mut(&mut root) else {
            bail!("eventsByWindowId is not an object");
        };
        let ctx = Ctx::new(allow);
        let mut changed = false;
        for (_, events) in obj.iter_mut() {
            if let Some(arr) = as_array_mut(events) {
                for event in arr.iter_mut() {
                    changed |= route_event(&ctx, event)?;
                }
            }
        }
        if !changed {
            return Ok(None);
        }
        Ok(Some(root.encode()))
    })
}

/// Scrub one JSONL line: a bare rrweb event object or the PostHog `["window_id", event]` tuple.
/// `Ok(Some(scrubbed))` = the line changed. `Ok(None)` = a recognized event the scrub policy left
/// unchanged (identical to the input). `Err` = could not anonymize; fail closed, drop the line.
///
/// "Unchanged" applies the production scrub policy, which deliberately passes some events through
/// untouched — DomContentLoaded/Load, and types/sources the router has no rule for — exactly as the
/// ingestion pipeline does (mirror data read offline was produced under this same policy). Callers
/// wanting a stricter policy than production must layer it themselves.
///
/// `line` is scratch — the in-place parse consumes it, so on `Ok(None)` keep a prior copy of the
/// bytes, not this buffer. Tuple lines come back with the wrapper intact and the window id
/// untouched. Anything that is not one of the two recognized shapes — a scalar, an array that is
/// not a `[string, object]` pair, or an object with no numeric rrweb `type` — is an `Err`, not a
/// silent pass-through: a scrubber must fail closed on input it doesn't understand rather than leave
/// the caller to guess whether unrecognized bytes were safe to keep. Invalid JSON is likewise an
/// `Err`.
pub fn anonymize_line(allow: &AllowLists, line: &mut [u8]) -> Result<Option<String>> {
    anonymize_line_with_ctx(&Ctx::new(allow), line)
}

/// Fail closed on an object that is not an rrweb event. Every rrweb event carries a numeric `type`;
/// an object without one is not something the scrubber recognizes, so — rather than pass it through
/// unscrubbed as `Ok(None)` — reject it (pass-through events like Load still have a `type`).
fn require_event_type(event: &Value<'_>) -> Result<()> {
    match as_object(event)
        .and_then(|o| o.get("type"))
        .and_then(as_small_uint)
    {
        Some(_) => Ok(()),
        None => bail!("jsonl line object has no numeric rrweb event `type`"),
    }
}

/// [`anonymize_line`] with a caller-owned [`Ctx`], so one image memo spans a whole session file
/// (an image recurring across lines is decoded once, like `anonymize_message` does
/// across a message's events). The cumulative cv decompression budget is per line — it resets on
/// every call, so a long session cannot starve later lines.
pub fn anonymize_line_with_ctx(ctx: &Ctx<'_>, line: &mut [u8]) -> Result<Option<String>> {
    contain_panics(|| {
        reject_if_too_deep(line, "jsonl line")?;
        let mut root = parse_untrusted(line).context("parse jsonl line")?;
        ctx.reset_cv_budget();
        let changed = match &mut root {
            Value::Object(_) => {
                require_event_type(&root)?;
                route_event(ctx, &mut root)?
            }
            Value::Array(items) => match items.as_mut_slice() {
                [Value::String(_), event @ Value::Object(_)] => {
                    require_event_type(event)?;
                    route_event(ctx, event)?
                }
                _ => bail!("jsonl line is an array but not a [window_id, event] tuple"),
            },
            _ => {
                bail!("jsonl line is neither an rrweb event object nor a [window_id, event] tuple")
            }
        };
        if !changed {
            return Ok(None);
        }
        Ok(Some(root.encode()))
    })
}

/// Convenience for tests/callers holding a single event as a JSON string: parse, scrub, re-serialize.
pub fn anonymize_event_str(allow: &AllowLists, event_json: &str) -> Result<String> {
    contain_panics(|| {
        let mut bytes = event_json.as_bytes().to_vec();
        let mut value = parse_untrusted(&mut bytes).context("parse event json")?;
        route_event(&Ctx::new(allow), &mut value)?;
        Ok(value.encode())
    })
}

/// Scrubs a single event in place, returning whether it changed. `Err` = "could not anonymize".
/// Builds its own `Ctx` (single-event scope); the message path uses [`anonymize_message`] instead so
/// the image memo is shared across events.
pub fn anonymize_event(allow: &AllowLists, event: &mut Value<'_>) -> Result<bool> {
    contain_panics(|| route_event(&Ctx::new(allow), event))
}

/// True when the event's `cv` marker means "compressed" (present and non-null).
#[doc(hidden)]
pub fn is_compressed_marker(v: Option<&Value<'_>>) -> bool {
    matches!(v, Some(v) if !matches!(v, Value::Static(StaticNode::Null)))
}

/// Internal routing shared by the entry points — consume [`anonymize_event`] or
/// [`anonymize_line`] instead; this may change signature in any release.
#[doc(hidden)]
pub fn route_event(ctx: &Ctx<'_>, event: &mut Value<'_>) -> Result<bool> {
    let Some(obj) = as_object_mut(event) else {
        return Ok(false);
    };
    let compressed = is_compressed_marker(obj.get("cv"));
    let ty = obj.get("type").and_then(as_small_uint);
    let Some(data) = obj.get_mut("data") else {
        return Ok(false);
    };
    route_data(ctx, ty, compressed, data)
}

/// Scrubs an event's `data` value in place given its routing shape (`type` + `cv` marker). This is the
/// single routing implementation shared by the tree walk and the streaming span-splice path: everything
/// the anonymizer changes lives inside `data`, so both paths agree by construction.
#[doc(hidden)]
pub fn route_data(
    ctx: &Ctx<'_>,
    ty: Option<u8>,
    compressed: bool,
    data: &mut Value<'_>,
) -> Result<bool> {
    match ty {
        Some(TYPE_FULL_SNAPSHOT) => {
            if compressed && matches!(data, Value::String(_)) {
                scrub_compressed_full_snapshot(ctx, data)
            } else {
                // Not actually whole-blob compressed — scrub as a plain object.
                Ok(scrub_full_snapshot(ctx, data))
            }
        }
        Some(TYPE_INCREMENTAL) => {
            let source = match as_object(data) {
                Some(d) => d.get("source").and_then(as_small_uint),
                None => return Ok(false),
            };
            match source {
                Some(SOURCE_MUTATION) => {
                    if compressed {
                        scrub_compressed_mutation(ctx, as_object_mut(data).unwrap())
                    } else {
                        Ok(scrub_mutation(ctx, data))
                    }
                }
                Some(SOURCE_INPUT) => Ok(scrub_text_field(
                    ctx.allow,
                    as_object_mut(data).unwrap(),
                    "text",
                )),
                Some(SOURCE_CANVAS_MUTATION) => Ok(scrub_canvas_mutation(ctx, data)),
                Some(SOURCE_STYLESHEET_RULE) => {
                    Ok(scrub_style_sheet_rule(ctx, as_object_mut(data).unwrap()))
                }
                Some(SOURCE_STYLE_DECLARATION) => {
                    Ok(scrub_style_declaration(ctx, as_object_mut(data).unwrap()))
                }
                Some(SOURCE_ADOPTED_STYLESHEET) => {
                    Ok(scrub_adopted_style_sheet(ctx, as_object_mut(data).unwrap()))
                }
                _ => Ok(false),
            }
        }
        Some(TYPE_META) => {
            // Meta `href` is the page URL — scrub path/query/fragment, keep the host.
            let Some(data) = as_object_mut(data) else {
                return Ok(false);
            };
            let Some(href) = data.get("href").and_then(as_str) else {
                return Ok(false);
            };
            match scrub_url(ctx, href) {
                Some(v) => {
                    data.insert(key("href"), string_value(v));
                    Ok(true)
                }
                None => Ok(false),
            }
        }
        Some(TYPE_CUSTOM) => match as_object_mut(data) {
            Some(data) if data.get("tag").and_then(as_str) == Some(JSON_LD_EVENT_TAG) => {
                let mut changed = scrub_json_ld_payload(data);
                if let Some(href) = data.get("href") {
                    if let Some(href) = as_str(href) {
                        if let Some(scrubbed) = scrub_url(ctx, href) {
                            data.insert(key("href"), string_value(scrubbed));
                            changed = true;
                        }
                    } else {
                        data.remove("href");
                        changed = true;
                    }
                }
                Ok(changed)
            }
            Some(data) => Ok(scrub_generic_field(ctx, data, "payload")),
            None => Ok(false),
        },
        Some(TYPE_PLUGIN) => {
            let Some(data) = as_object_mut(data) else {
                return Ok(false);
            };
            let plugin = data.get("plugin").and_then(as_str).map(str::to_string);
            Ok(match plugin.as_deref() {
                Some(NETWORK_PLUGIN) => scrub_network_plugin(ctx, data, "payload"),
                Some(CONSOLE_PLUGIN) => scrub_console_plugin(ctx, data, "payload"),
                _ => scrub_generic_field(ctx, data, "payload"),
            })
        }
        // DomContentLoaded, Load, unknown types: pass-through.
        _ => Ok(false),
    }
}

fn scrub_text_field(allow: &AllowLists, obj: &mut Object<'_>, field: &'static str) -> bool {
    let Some(cur) = obj.get(field).and_then(as_str) else {
        return false;
    };
    match scrub_text(allow, cur) {
        Some(v) => {
            obj.insert(key(field), string_value(v));
            true
        }
        None => false,
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn message_scrubs_every_window() {
        let allow = AllowLists::new(["hello", "here"], Vec::<String>::new());
        let mut json = br#"{"w1":[{"type":3,"data":{"source":5,"id":1,"text":"hello secret","isChecked":false}}],"w2":[{"type":3,"data":{"source":5,"id":2,"text":"world here","isChecked":false}}]}"#.to_vec();
        let out = anonymize_message(&allow, &mut json)
            .unwrap()
            .expect("something changed");
        let v: serde_json::Value = serde_json::from_str(&out).unwrap();
        assert_eq!(v["w1"][0]["data"]["text"], "hello ******");
        assert_eq!(v["w2"][0]["data"]["text"], "***** here");
    }

    #[test]
    fn unchanged_message_returns_none() {
        let allow = AllowLists::new(Vec::<String>::new(), Vec::<String>::new());
        // A Load event (type 1) is pass-through; the caller keeps its original parse.
        let mut json = br#"{"w1":[{"type":1,"data":{}}]}"#.to_vec();
        assert!(anonymize_message(&allow, &mut json).unwrap().is_none());
    }

    #[test]
    fn line_scrubs_a_tuple_keeping_the_wrapper_and_window_id() {
        let allow = AllowLists::new(["keep"], Vec::<String>::new());
        let mut line =
            br#"["w-1",{"type":3,"data":{"source":5,"id":1,"text":"keep secret","isChecked":false}}]"#
                .to_vec();
        let out = anonymize_line(&allow, &mut line)
            .unwrap()
            .expect("the text changes");
        assert!(out.starts_with(r#"["w-1","#), "wrapper intact: {out}");
        let v: serde_json::Value = serde_json::from_str(&out).unwrap();
        assert_eq!(v[1]["data"]["text"], "keep ******");
    }

    #[test]
    fn line_scrubs_a_bare_event_like_its_tuple_twin() {
        let allow = AllowLists::new(["keep"], Vec::<String>::new());
        let event =
            r#"{"type":3,"data":{"source":5,"id":1,"text":"keep secret","isChecked":false}}"#;

        let mut bare = event.as_bytes().to_vec();
        let bare_out = anonymize_line(&allow, &mut bare).unwrap().unwrap();

        let mut tuple = format!(r#"["w-1",{event}]"#).into_bytes();
        let tuple_out = anonymize_line(&allow, &mut tuple).unwrap().unwrap();

        let tuple_v: serde_json::Value = serde_json::from_str(&tuple_out).unwrap();
        let bare_v: serde_json::Value = serde_json::from_str(&bare_out).unwrap();
        assert_eq!(bare_v, tuple_v[1]);
    }

    #[test]
    fn line_returns_none_only_for_unchanged_recognized_events() {
        let allow = AllowLists::new(Vec::<String>::new(), Vec::<String>::new());
        // Recognized events that need no scrubbing (type 1 is pass-through), bare and tuple-wrapped.
        for line in [
            br#"{"type":1,"data":{}}"#.as_slice(),
            br#"["w-1",{"type":1,"data":{}}]"#.as_slice(),
        ] {
            let mut bytes = line.to_vec();
            assert!(
                anonymize_line(&allow, &mut bytes).unwrap().is_none(),
                "expected Ok(None) for {}",
                String::from_utf8_lossy(line)
            );
        }
    }

    #[test]
    fn line_fails_closed_on_unrecognized_shapes_and_invalid_json() {
        let allow = AllowLists::new(Vec::<String>::new(), Vec::<String>::new());
        // A scrubber must not silently pass through input it doesn't recognize as an event.
        for line in [
            br#"["w-1","not an event"]"#.as_slice(), // tuple with a non-object second element
            br#"[{"type":1},{"type":1}]"#.as_slice(), // array of events, not a [window_id, event] pair
            br#"["w-1",{"type":1},{"extra":1}]"#.as_slice(), // three-element array
            br#"{"foo":"bar"}"#.as_slice(),           // object with no rrweb event `type`
            br#"["w-1",{"foo":"bar"}]"#.as_slice(),   // tuple wrapping a non-event object
            br#"42"#.as_slice(),                      // scalar
            br#""just a string""#.as_slice(),         // scalar
            br#"{not json"#.as_slice(),               // invalid JSON
        ] {
            let mut bytes = line.to_vec();
            assert!(
                anonymize_line(&allow, &mut bytes).is_err(),
                "expected Err for {}",
                String::from_utf8_lossy(line)
            );
        }
    }

    #[test]
    fn line_streaming_shares_the_image_memo_across_lines() {
        use crate::testkit::png_data_uri;
        let allow = AllowLists::new(Vec::<String>::new(), Vec::<String>::new());
        let ctx = Ctx::new(&allow);
        let uri = png_data_uri(40, 40, [12, 34, 56, 255]);
        let line = serde_json::to_string(&serde_json::json!({
            "type": 2,
            "data": {
                "node": { "type": 0, "childNodes": [
                    { "type": 2, "tagName": "div", "attributes": { "rr_dataURL": uri }, "childNodes": [] }
                ]},
                "initialOffset": { "top": 0, "left": 0 }
            }
        }))
        .unwrap();

        let mut first = line.clone().into_bytes();
        let mut second = line.into_bytes();
        let first_out = anonymize_line_with_ctx(&ctx, &mut first).unwrap().unwrap();
        let second_out = anonymize_line_with_ctx(&ctx, &mut second).unwrap().unwrap();
        assert_eq!(
            first_out, second_out,
            "a recurring image must scrub identically through the shared memo"
        );
    }

    #[test]
    fn line_resets_the_cv_budget_per_call() {
        let allow = AllowLists::new(Vec::<String>::new(), Vec::<String>::new());
        let ctx = Ctx::new(&allow);
        let texts_gz = crate::cv::bytes_to_latin1(
            &crate::compression::gzip(br#"[{"id":5,"value":"topsecret stuff"}]"#).unwrap(),
        );
        let line = serde_json::to_string(&serde_json::json!(
            { "type": 3, "cv": "2024-10", "data": { "source": 0, "texts": texts_gz } }
        ))
        .unwrap();

        // A prior line exhausted the budget; the next call must start from a fresh one.
        ctx.cv_budget.set(0);
        let mut bytes = line.into_bytes();
        let out = anonymize_line_with_ctx(&ctx, &mut bytes)
            .unwrap()
            .expect("the cv payload decompresses and scrubs");
        assert!(out.contains("source"), "still a mutation event: {out}");
    }

    #[test]
    fn recurring_inline_image_is_neutralized_across_windows() {
        // An inline image can recur across events/windows in one message (the case the per-message image
        // memo targets). Every occurrence must be neutralized to a data-image — never the raw original —
        // and identical inputs must yield identical output. This drives the real `anonymize_message`
        // entry (route -> full snapshot -> dom -> image scrub).
        use crate::testkit::png_data_uri;
        use serde_json::json;
        let allow = AllowLists::new(Vec::<String>::new(), Vec::<String>::new());
        let uri = png_data_uri(40, 40, [12, 34, 56, 255]);
        let snapshot = json!({
            "type": 2,
            "data": {
                "node": { "type": 0, "childNodes": [
                    { "type": 2, "tagName": "div", "attributes": { "rr_dataURL": uri.clone() }, "childNodes": [] }
                ]},
                "initialOffset": { "top": 0, "left": 0 }
            }
        });
        let message = json!({ "w1": [snapshot.clone()], "w2": [snapshot] });
        let mut bytes = serde_json::to_vec(&message).unwrap();
        let out = anonymize_message(&allow, &mut bytes)
            .unwrap()
            .expect("the image should change");
        let v: serde_json::Value = serde_json::from_str(&out).unwrap();
        let img = |w: &str| {
            v[w][0]["data"]["node"]["childNodes"][0]["attributes"]["rr_dataURL"]
                .as_str()
                .unwrap()
                .to_string()
        };
        assert_eq!(
            img("w1"),
            crate::data_uri::blank_image_data_uri(),
            "blanked without the scrub lane"
        );
        assert_eq!(
            img("w1"),
            img("w2"),
            "a recurring image must be neutralized consistently"
        );
    }
}