:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
//! Length-preserving text scrub: numeric tokens -> `#`, allow-listed words kept, everything else -> `*`.
//! `None` means "unchanged" (caller keeps the original).
use crate::allow_lists::AllowLists;
/// Replacement char for redacted (non-allow-listed) word characters.
pub const REDACT_CHAR: char = '*';
/// Replacement char for numeric tokens.
pub const NUMBER_CHAR: char = '#';
/// Guaranteed email-PII pass. Hand-rolled scan (expand around each `@`, equivalent to
/// `/[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g`) because that regex backtracks
/// quadratically on long unbroken charset runs (base64 bodies: minutes per event).
pub fn redact_emails(input: &str) -> Option<String> {
let bytes = input.as_bytes();
let mut at = find_at(bytes, 0)?;
let mut out = String::new();
let mut last = 0usize;
let mut changed = false;
loop {
let mut start = at;
while start > last && is_local_char(bytes[start - 1]) {
start -= 1;
}
let mut scan_end = at + 1;
while scan_end < bytes.len() && is_domain_char(bytes[scan_end]) {
scan_end += 1;
}
let end = trim_domain_to_tld(bytes, at + 1, scan_end);
if start < at {
if let Some(end) = end {
out.push_str(&input[last..start]);
for _ in input[start..end].chars() {
out.push(REDACT_CHAR);
}
last = end;
changed = true;
match find_at(bytes, end) {
Some(n) => at = n,
None => break,
}
continue;
}
}
match find_at(bytes, at + 1) {
Some(n) => at = n,
None => break,
}
}
if !changed {
return None;
}
out.push_str(&input[last..]);
Some(out)
}
fn find_at(bytes: &[u8], from: usize) -> Option<usize> {
memchr::memchr(b'@', &bytes[from..]).map(|i| from + i)
}
/// Longest `end` in `(domain_start, scan_end]` where the domain ends with `.` + >=2 letters, else None.
fn trim_domain_to_tld(bytes: &[u8], domain_start: usize, scan_end: usize) -> Option<usize> {
let mut i = scan_end;
while i > domain_start {
let mut letters = 0usize;
while i > domain_start && is_ascii_letter(bytes[i - 1]) {
i -= 1;
letters += 1;
}
if letters >= 2 && i > domain_start + 1 && bytes[i - 1] == b'.' {
return Some(i + letters);
}
i -= 1; // skip the non-letter (or short-TLD dot) and keep looking left
}
None
}
fn is_ascii_letter(c: u8) -> bool {
c.is_ascii_alphabetic()
}
// [A-Za-z0-9._%+-]
fn is_local_char(c: u8) -> bool {
c.is_ascii_alphanumeric() || matches!(c, b'.' | b'_' | b'%' | b'+' | b'-')
}
// [A-Za-z0-9.-]
fn is_domain_char(c: u8) -> bool {
c.is_ascii_alphanumeric() || matches!(c, b'.' | b'-')
}
pub fn scrub_text(allow: &AllowLists, input: &str) -> Option<String> {
// Nuke whole email addresses first; the tokenizer then runs over what's left.
let emails = redact_emails(input);
let text: &str = emails.as_deref().unwrap_or(input);
match tokenize_scrub(allow, text) {
Some(t) => Some(t),
None => emails,
}
}
fn tokenize_scrub(allow: &AllowLists, text: &str) -> Option<String> {
let bytes = text.as_bytes();
let mut out = String::with_capacity(text.len());
let mut changed = false;
let mut i = 0usize;
while i < bytes.len() {
// Non-word run: find its end bytewise (ASCII fast path) and copy it in one slice push.
let run_start = i;
while i < bytes.len() {
let b = bytes[i];
if b < 0x80 {
if is_ascii_word_byte(b) {
break;
}
i += 1;
} else {
let c = next_char(text, i);
if is_word_char(c) {
break;
}
i += c.len_utf8();
}
}
out.push_str(&text[run_start..i]);
if i >= bytes.len() {
break;
}
// Word run; numeric-token shape is classified in the same scan.
let word_start = i;
let mut saw_digit = false;
let mut numericish = true; // only digits and . , - + so far
while i < bytes.len() {
let b = bytes[i];
if b < 0x80 {
if !is_ascii_word_byte(b) {
break;
}
if b.is_ascii_digit() {
saw_digit = true;
} else if !matches!(b, b'.' | b',' | b'-' | b'+') {
numericish = false;
}
i += 1;
} else {
let c = next_char(text, i);
if !is_word_char(c) {
break;
}
numericish = false;
i += c.len_utf8();
}
}
emit_word(
allow,
&text[word_start..i],
saw_digit && numericish,
&mut out,
&mut changed,
);
}
if changed {
Some(out)
} else {
None
}
}
fn next_char(text: &str, at: usize) -> char {
text[at..].chars().next().expect("at is a char boundary")
}
// A "word" is a maximal run of word chars: Unicode letters/numbers, `_`, `'`, `’`.
// The byte test is the ASCII projection of `is_word_char` (`’` is non-ASCII, handled per-char).
fn is_ascii_word_byte(b: u8) -> bool {
b.is_ascii_alphanumeric() || b == b'_' || b == b'\''
}
fn is_word_char(c: char) -> bool {
c.is_alphanumeric() || c == '_' || c == '\'' || c == '\u{2019}'
}
fn emit_word(allow: &AllowLists, word: &str, numeric: bool, out: &mut String, changed: &mut bool) {
if numeric {
push_redacted(word, NUMBER_CHAR, out);
*changed = true;
} else if word_is_allowed(allow, word) {
out.push_str(word);
} else {
push_redacted(word, REDACT_CHAR, out);
*changed = true;
}
}
/// Length-preserving redaction: one mark per source code point, pushed in chunks.
fn push_redacted(word: &str, mark: char, out: &mut String) {
const STARS: &str = "********************************";
const HASHES: &str = "################################";
let chunk = if mark == NUMBER_CHAR { HASHES } else { STARS };
let mut n = if word.is_ascii() {
word.len()
} else {
word.chars().count()
};
while n > 0 {
let take = n.min(chunk.len());
out.push_str(&chunk[..take]);
n -= take;
}
}
fn word_is_allowed(allow: &AllowLists, word: &str) -> bool {
if allow.text_contains(word) {
return true;
}
// Possessive/quote forms all end in `s`, `'` or `\u{2019}` (whose UTF-8 ends 0x99) — anything
// else can't match below, so the suffix machinery is skipped for the typical redacted word.
if !matches!(
word.as_bytes().last(),
Some(b's') | Some(b'\'') | Some(0x99)
) {
return false;
}
if !word.is_ascii() && word.contains('\u{2019}') {
let normalized = word.replace('\u{2019}', "'");
if allow.text_contains(&normalized) {
return true;
}
if let Some(base) = strip_possessive(&normalized) {
if allow.text_contains(base) {
return true;
}
}
}
if let Some(base) = strip_possessive(word) {
if allow.text_contains(base) {
return true;
}
}
false
}
fn strip_possessive(word: &str) -> Option<&str> {
for suffix in ["'s", "\u{2019}s", "'", "\u{2019}"] {
if let Some(base) = word.strip_suffix(suffix) {
if !base.is_empty() {
return Some(base);
}
}
}
None
}