:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
//! Case-insensitive allow lists of text words and URL path segments kept verbatim by the scrubbers.
//! Mirrors `anonymize/allow-lists.ts`.
// The lookups run once per word token of every text node; ahash beats SipHash on short strings and
// the sets are built from operator-controlled config, so hash-DoS resistance buys nothing here.
use ahash::AHashSet;
use anyhow::{Context, Result};
use serde::Deserialize;
/// Bounds on the *retained* allow-list, mirroring the TS loader's hygiene
/// (`anonymize/allow-list-loader.ts`). These cap what is kept, not peak parse memory — the JSON is
/// fully materialized by `serde_json` first. That's acceptable because the document is
/// operator-controlled config loaded once at startup, not per-request untrusted input.
const MAX_ALLOW_LIST_ENTRIES: usize = 500_000;
const MAX_ALLOW_LIST_ENTRY_LEN: usize = 256;
// The fail-safe default lists, embedded so offline consumers with no access to the shipped allow
// lists scrub with production-equivalent vocabulary instead of redacting everything. The `.txt`
// files under `crosslang/` are the single source of truth: the crate embeds them here, and the TS
// pipeline's `anonymize/default-dict.ts` mirrors them (a Jest test reads these files and asserts the
// match, so the two stay in sync without either side parsing the other's source). See
// `crosslang/README.md` for the cross-language folder convention.
const DEFAULT_TEXT_WORDS: &str = include_str!("crosslang/default_text_words.txt");
const DEFAULT_URL_SEGMENTS: &str = include_str!("crosslang/default_url_segments.txt");
fn has_upper_ascii(s: &str) -> bool {
s.bytes().any(|b| b.is_ascii_uppercase())
}
fn ascii_lowercase(s: &str) -> String {
// Only ASCII A-Z is folded, exactly like the TS (which does not lowercase non-ASCII).
let mut out = String::with_capacity(s.len());
for ch in s.chars() {
if ch.is_ascii_uppercase() {
out.push(ch.to_ascii_lowercase());
} else {
out.push(ch);
}
}
out
}
#[derive(Debug)]
pub struct AllowLists {
text: AHashSet<String>,
url: AHashSet<String>,
/// Bit i set = the text set contains a word of byte length i (lengths >= 63 share bit 63).
/// Most redacted tokens miss the mask and skip the hash entirely.
text_lens: u64,
url_lens: u64,
}
fn len_bit(s: &str) -> u64 {
1u64 << s.len().min(63)
}
/// The production default lists (English stopwords/UI vocabulary + safe URL segments) — the same
/// fallback the TS pipeline uses when the shipped lists can't be loaded. Consumers with explicit
/// lists should prefer [`AllowLists::from_json_bytes`]; [`AllowLists::new`] stays available for
/// deliberate empty construction (redact everything).
impl Default for AllowLists {
fn default() -> Self {
Self::new(DEFAULT_TEXT_WORDS.lines(), DEFAULT_URL_SEGMENTS.lines())
}
}
#[derive(Deserialize)]
struct RawAllowLists {
#[serde(default)]
text: Option<serde_json::Value>,
#[serde(default)]
url: Option<serde_json::Value>,
}
/// The TS loader's sanitize pass: keep only string entries within the length bound, cap the count.
fn sanitize_entries(raw: Option<serde_json::Value>) -> Vec<String> {
let Some(serde_json::Value::Array(items)) = raw else {
return Vec::new();
};
items
.into_iter()
.filter_map(|v| match v {
serde_json::Value::String(s) if s.len() <= MAX_ALLOW_LIST_ENTRY_LEN => Some(s),
_ => None,
})
.take(MAX_ALLOW_LIST_ENTRIES)
.collect()
}
impl AllowLists {
pub fn new<I, J>(text: I, url: J) -> Self
where
I: IntoIterator,
I::Item: AsRef<str>,
J: IntoIterator,
J::Item: AsRef<str>,
{
let text: AHashSet<String> = text
.into_iter()
.map(|w| ascii_lowercase(w.as_ref()))
.collect();
let url: AHashSet<String> = url
.into_iter()
.map(|s| ascii_lowercase(s.as_ref()))
.collect();
let text_lens = text.iter().map(|w| len_bit(w)).fold(0, |a, b| a | b);
let url_lens = url.iter().map(|w| len_bit(w)).fold(0, |a, b| a | b);
Self {
text,
url,
text_lens,
url_lens,
}
}
/// Parse the `{ "text": [...], "url": [...] }` allow-list document PostHog ships (the shape
/// the Node addon loads from S3), with the TS loader's sanitize limits: non-string and
/// over-long entries are dropped, each list is capped at 500k entries. Missing or non-array
/// fields become empty lists (fail-safe: over-redaction, never under-redaction); only invalid
/// JSON is an error.
pub fn from_json_bytes(bytes: &[u8]) -> Result<Self> {
let raw: RawAllowLists = serde_json::from_slice(bytes).context("parse allow-lists json")?;
Ok(Self::new(
sanitize_entries(raw.text),
sanitize_entries(raw.url),
))
}
pub fn text_contains(&self, word: &str) -> bool {
if self.text_lens & len_bit(word) == 0 {
return false;
}
if has_upper_ascii(word) {
self.text.contains(&ascii_lowercase(word))
} else {
self.text.contains(word)
}
}
pub fn url_contains(&self, segment: &str) -> bool {
if self.url_lens & len_bit(segment) == 0 {
return false;
}
if has_upper_ascii(segment) {
self.url.contains(&ascii_lowercase(segment))
} else {
self.url.contains(segment)
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn from_json_bytes_sanitizes_like_the_ts_loader() {
let long = "x".repeat(MAX_ALLOW_LIST_ENTRY_LEN + 1);
let doc = serde_json::json!({
"text": ["Hello", 42, null, long, "world"],
"url": ["Api"],
});
let allow = AllowLists::from_json_bytes(&serde_json::to_vec(&doc).unwrap()).unwrap();
// Kept entries match case-insensitively, exactly like `AllowLists::new`.
assert!(allow.text_contains("hello") && allow.text_contains("HELLO"));
assert!(allow.text_contains("world"));
assert!(allow.url_contains("api") && allow.url_contains("API"));
// Non-string and over-long entries are dropped, not kept and not fatal.
assert!(!allow.text_contains(&long));
assert!(!allow.text_contains("42"));
// Missing/non-array fields fail safe to empty lists; only invalid JSON errors.
let empty = AllowLists::from_json_bytes(br#"{"text": "not an array"}"#).unwrap();
assert!(!empty.text_contains("hello"));
assert!(AllowLists::from_json_bytes(b"{nope").is_err());
}
#[test]
fn defaults_allow_stopwords_and_still_redact_names() {
let allow = AllowLists::default();
// A word from the TS `DEFAULT_TEXT_WORDS` and a segment from `DEFAULT_URL_SEGMENTS`.
assert!(allow.text_contains("the"));
assert!(allow.url_contains("api"));
assert_eq!(
crate::text::scrub_text(&allow, "John Fakename accepted the request"),
Some("**** ******** accepted the request".to_string()),
"names must still redact under the default lists"
);
}
}