Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

9.8 KB
Raw Blame
//! Node runs these rules with RE2, so `translate_re2` gives the `regex` crate the same semantics.

use std::sync::LazyLock;

use regex::Regex;

use crate::avro::AvroValue;
use crate::js::{json_stringify, JsValue};
use crate::record::LogRecord;

pub const PII_REDACTED: &str = "{{REDACTED}}";

#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct PiiScrubStats {
    pub pii_replacements: u64,
}

pub const EMPTY_PII: PiiScrubStats = PiiScrubStats {
    pii_replacements: 0,
};

pub fn encode_attribute_cell(semantic: &str) -> String {
    json_stringify(&JsValue::string(semantic))
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct PiiRule {
    pub name: &'static str,
    /// The pattern text exactly as the TypeScript gives it to RE2.
    pub pattern: &'static str,
    pub replacement: &'static str,
}

/// The scrub reads the fired rule from the group index, and the earlier alternative wins a tie at one offset.
pub const PII_RULES: [PiiRule; 7] = [
    PiiRule {
        name: "bearer",
        pattern: r"(?i:Bearer\s+[-A-Za-z0-9._~+/]+=*)",
        replacement: "Bearer {{REDACTED}}",
    },
    PiiRule {
        name: "stripe",
        pattern: r"\bsk_(?:live|test)_[a-zA-Z0-9]{20,}\b",
        replacement: PII_REDACTED,
    },
    PiiRule {
        name: "github",
        pattern: r"\bgh[pousr]_[A-Za-z0-9]{20,}\b",
        replacement: PII_REDACTED,
    },
    PiiRule {
        name: "slack",
        pattern: r"\bxox[abeprs]-[A-Za-z0-9-]{10,}\b",
        replacement: PII_REDACTED,
    },
    PiiRule {
        name: "aws_access_key_id",
        pattern: r"\b(?:AKIA|ASIA)[0-9A-Z]{16}\b",
        replacement: PII_REDACTED,
    },
    PiiRule {
        name: "jwt",
        pattern: r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{4,}\.[A-Za-z0-9_-]*",
        replacement: PII_REDACTED,
    },
    PiiRule {
        name: "email",
        pattern: r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b",
        replacement: PII_REDACTED,
    },
];

/// RE2 keeps `\d`, `\w`, `\s` and `\b` ASCII in UTF-8 mode, and its `\s` excludes `\v`, so these become explicit classes.
/// The rules use none of these escapes inside a bracket class, so the rewrite does not track classes.
pub fn translate_re2(pattern: &str) -> String {
    let mut out = String::with_capacity(pattern.len() + 16);
    let mut chars = pattern.chars();
    while let Some(c) = chars.next() {
        if c != '\\' {
            out.push(c);
            continue;
        }
        match chars.next() {
            Some('d') => out.push_str("[0-9]"),
            Some('w') => out.push_str("[0-9A-Za-z_]"),
            Some('s') => out.push_str(r"[\t\n\f\r ]"),
            Some('b') => out.push_str(r"(?-u:\b)"),
            Some(other) => {
                out.push('\\');
                out.push(other);
            }
            None => out.push('\\'),
        }
    }
    out
}

pub fn combined_regex(patterns: impl IntoIterator<Item = &'static str>) -> Regex {
    let joined = patterns
        .into_iter()
        .map(|p| format!("({})", translate_re2(p)))
        .collect::<Vec<_>>()
        .join("|");
    regex::RegexBuilder::new(&joined)
        .size_limit(64 << 20)
        .dfa_size_limit(64 << 20)
        .build()
        .expect("combined rule regex compiles")
}

/// The DFA reports the matching rule with the alternation's priority, so no capture search runs.
pub struct RuleRegex(regex_automata::meta::Regex);

impl RuleRegex {
    pub fn new(patterns: impl IntoIterator<Item = &'static str>) -> Self {
        let translated: Vec<String> = patterns.into_iter().map(translate_re2).collect();
        let regex = regex_automata::meta::Regex::builder()
            .configure(
                regex_automata::meta::Config::new()
                    .nfa_size_limit(Some(64 << 20))
                    .dfa_size_limit(Some(64 << 20))
                    .hybrid_cache_capacity(64 << 20),
            )
            .build_many(&translated)
            .expect("rule regex compiles");
        Self(regex)
    }

    pub fn matches<'h>(
        &'h self,
        haystack: &'h str,
    ) -> impl Iterator<Item = (usize, usize, usize)> + 'h {
        self.0
            .find_iter(haystack)
            .map(|m| (m.start(), m.end(), m.pattern().as_usize()))
    }
}

static PII_RULE_RE: LazyLock<RuleRegex> =
    LazyLock::new(|| RuleRegex::new(PII_RULES.iter().map(|r| r.pattern)));

fn scrub_str(input: &str) -> (Option<String>, u64) {
    let mut replacements = 0u64;
    let mut out: Option<String> = None;
    let mut copied_up_to = 0;
    for (start, end, rule) in PII_RULE_RE.matches(input) {
        if start == end {
            continue;
        }
        replacements += 1;
        let buf = out.get_or_insert_with(|| String::with_capacity(input.len()));
        buf.push_str(&input[copied_up_to..start]);
        buf.push_str(PII_RULES[rule].replacement);
        copied_up_to = end;
    }
    if let Some(buf) = &mut out {
        buf.push_str(&input[copied_up_to..]);
    }
    (out, replacements)
}

/// The count includes only matches that a rule group claims. No rule can match an empty string.
pub fn scrub_plain_string_with_stats(input: &str) -> (String, u64) {
    let (out, replacements) = scrub_str(input);
    (out.unwrap_or_else(|| input.to_string()), replacements)
}

pub fn scrub_plain_string(input: &str) -> String {
    scrub_plain_string_with_stats(input).0
}

/// The Avro decoder writes U+FFFD for invalid bytes, so no lone-surrogate handling is needed here.
pub fn scrub_log_record(record: &mut LogRecord) -> PiiScrubStats {
    let mut total = 0u64;

    if let Some(body) = record.body() {
        let (output, n) = scrub_str(body);
        total += n;
        if let Some(output) = output {
            record.set("body", AvroValue::String(output));
        }
    }

    // Keys and their order do not change, so each value is replaced in place.
    if let Some(attributes) = record.attributes_mut() {
        for value in attributes.values_mut() {
            if let AvroValue::String(s) = value {
                let (output, n) = scrub_str(s);
                total += n;
                if let Some(output) = output {
                    *s = output;
                }
            }
        }
    }

    PiiScrubStats {
        pii_replacements: total,
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn members(pattern: &str, range: std::ops::Range<u32>) -> Vec<u32> {
        let re = Regex::new(&format!("^(?:{})$", translate_re2(pattern))).unwrap();
        let mut buf = [0u8; 4];
        range
            .filter_map(char::from_u32)
            .filter(|c| re.is_match(c.encode_utf8(&mut buf)))
            .map(|c| c as u32)
            .collect()
    }

    fn non_ascii_members(pattern: &str) -> Vec<u32> {
        members(pattern, 0x80..0x110000)
    }

    /// Memberships recorded from node-re2 1.22.1 over every code point.
    #[test]
    fn class_membership_matches_node_re2() {
        assert_eq!(non_ascii_members("(?i:[-A-Za-z0-9._~+/])"), [0x17F, 0x212A]);
        assert!(non_ascii_members(r"(?i:[Bbear])").is_empty());
        assert!(non_ascii_members(r"\d").is_empty());
        assert!(non_ascii_members(r"\w").is_empty());
        assert!(non_ascii_members(r"\s").is_empty());
        let ascii_space = members(r"\s", 0..0x80);
        assert_eq!(ascii_space, [0x09, 0x0A, 0x0C, 0x0D, 0x20]);
    }

    #[test]
    fn rule_regex_matches_the_alternation() {
        const PARTS: &[&str] = &[
            "2025-01-15T12:00:00.123Z",
            "I0115 12:00:00.5",
            "Mon Jan  5 12:00:00 UTC 2025",
            "[email protected]",
            "usr_AbC123",
            "cus_ABcdefghijklmn",
            "_1234_5",
            "_3fa85f64-5717-4562-b3fc-2c963f66afa6",
            "ip-10-0-0-1.internal",
            "0xBEEF",
            "192.168.1.20",
            "deadbeefcafe",
            "Bearer a.b~c==",
            "bearer \u{212A}x",
            "sk_live_ABCDEFGHIJKLMNOPQRSTUV",
            "ghp_ABCDEFGHIJKLMNOPQRSTUVWXYZ",
            "xoxb-1234567890-a",
            "AKIAABCDEFGHIJKLMNOP",
            "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxIn0.s",
            "é",
            "_",
            "-",
            ".",
            " ",
            "x",
            "12",
        ];
        let rule_sets: [&[&str]; 2] = [
            &PII_RULES.map(|r| r.pattern),
            &crate::pattern::MASK_RULES.map(|r| r.pattern),
        ];
        for patterns in rule_sets {
            let alternation = combined_regex(patterns.iter().copied());
            let rules = RuleRegex::new(patterns.iter().copied());
            let mut state = 0x9e37_79b9_7f4a_7c15u64;
            for _ in 0..20_000 {
                let mut input = String::new();
                for _ in 0..state % 8 {
                    state ^= state << 13;
                    state ^= state >> 7;
                    state ^= state << 17;
                    input.push_str(PARTS[(state % PARTS.len() as u64) as usize]);
                }
                let expected: Vec<_> = alternation
                    .captures_iter(&input)
                    .map(|c| {
                        let whole = c.get(0).unwrap();
                        let rule = (0..patterns.len()).find(|&i| c.get(i + 1).is_some());
                        (whole.start(), whole.end(), rule.unwrap())
                    })
                    .collect();
                assert_eq!(
                    rules.matches(&input).collect::<Vec<_>>(),
                    expected,
                    "{input:?}"
                );
            }
        }
    }

    #[test]
    fn re2_edge_cases_match_node() {
        assert_eq!(
            scrub_plain_string("bearer \u{212A}abc").as_str(),
            "Bearer {{REDACTED}}"
        );
        assert_eq!(scrub_plain_string("Bearer\u{000B}ab"), "Bearer\u{000B}ab");
        let re = Regex::new(&translate_re2(r"\bab\b")).unwrap();
        assert_eq!(re.replace_all("éab abé", "X"), "éX Xé");
    }
}