Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
//! Fixed-rule discovery of credentials in text and named fields.
//! Positions are Unicode character offsets, suitable for Python string slicing.
use regex::Regex;
use serde::Deserialize;
use std::collections::{HashMap, HashSet};
#[derive(Deserialize)]
struct Config {
normalized_names: HashSet<String>,
suffixes: Vec<String>,
token_hints: Vec<String>,
token_pattern: String,
assignment_pattern: String,
cli_pattern: String,
auth_pattern: String,
scheme_pattern: String,
bracket_pattern: String,
reference_pattern: String,
}
/// Compiled, immutable fixed rules used for Session credential discovery.
pub struct PrivacyExtractor {
cfg: Config,
token: Regex,
assignment: Regex,
cli: Regex,
auth: Regex,
scheme: Regex,
bracket: Regex,
reference: Regex,
bold: Regex,
code_delimiter: Regex,
unicode_alphanumeric: Regex,
}
#[derive(Clone, Debug)]
struct Hit {
start: usize,
end: usize,
field: String,
}
impl PrivacyExtractor {
/// Compile the built-in credential rules once for reuse across messages.
pub fn new() -> Result<Self, String> {
let cfg: Config = serde_json::from_str(include_str!("privacy_extractor_rules.json"))
.map_err(|error| error.to_string())?;
Ok(Self {
token: Regex::new(&cfg.token_pattern).map_err(|error| error.to_string())?,
assignment: Regex::new(&format!("(?i){}", cfg.assignment_pattern))
.map_err(|error| error.to_string())?,
cli: Regex::new(&cfg.cli_pattern).map_err(|error| error.to_string())?,
auth: Regex::new(&cfg.auth_pattern).map_err(|error| error.to_string())?,
scheme: Regex::new(&cfg.scheme_pattern).map_err(|error| error.to_string())?,
bracket: Regex::new(&cfg.bracket_pattern).map_err(|error| error.to_string())?,
reference: Regex::new(&cfg.reference_pattern).map_err(|error| error.to_string())?,
bold: Regex::new(r"(?:^|[\s(>=:])\*\*(?P<text>[^\s*](?:[^\r\n]*?[^\s*])?)\*\*")
.map_err(|error| error.to_string())?,
code_delimiter: Regex::new(r"`+|~{3,}").map_err(|error| error.to_string())?,
unicode_alphanumeric: Regex::new(r"^[\p{L}\p{N}]$")
.map_err(|error| error.to_string())?,
cfg,
})
}
fn word(&self, c: char) -> bool {
if c.is_ascii() {
return c.is_ascii_alphanumeric() || c == '_';
}
// General categories Letter and Number match Python's alphanumeric
// definition. Rust's Alphabetic property also includes combining marks.
self.unicode_alphanumeric
.is_match(c.encode_utf8(&mut [0; 4]))
}
fn boundary(&self, s: &str, pos: usize) -> bool {
s[..pos]
.chars()
.next_back()
.map_or(true, |c| c != '-' && !self.word(c))
}
/// Whether a structured input key denotes a complete sensitive value.
pub fn is_sensitive_key(&self, key: &str) -> bool {
let lower = key.to_lowercase().replace('-', "_");
self.cfg
.suffixes
.iter()
.any(|suffix| lower.ends_with(suffix))
|| self.cfg.normalized_names.contains(&normalize_key(&lower))
}
fn env_end(&self, s: &str, p: usize) -> Option<usize> {
let tail = &s[p..];
if tail.starts_with("${") {
let mut n = 0;
for (i, c) in tail[2..].char_indices() {
if c == '}' {
return (n > 0).then_some(p + 2 + i + 1);
}
if c == '{' || space(c) {
return None;
}
n += 1;
}
} else if tail.starts_with('$') {
let mut chars = tail[1..].char_indices();
let (_, first) = chars.next()?;
if !first.is_ascii_alphabetic() && first != '_' {
return None;
}
let mut end = p + 1 + first.len_utf8();
for (i, c) in chars {
if !self.word(c) {
break;
}
end = p + 1 + i + c.len_utf8();
}
return Some(end);
} else if tail.starts_with('<') {
for (i, c) in tail[1..].char_indices() {
if c == '<' {
return None;
}
if c == '>' {
return (i > 0).then_some(p + 1 + i + 1);
}
}
}
None
}
fn full_reference(&self, s: &str) -> bool {
self.reference
.find(s)
.is_some_and(|m| m.start() == 0 && m.end() == s.len())
}
fn add(
&self,
hits: &mut Vec<Hit>,
s: &str,
start: usize,
end: usize,
key: &str,
structured: bool,
) -> Result<(), String> {
let value = &s[start..end];
if value.is_empty()
|| (!structured && (value == "|" || value == ">"))
|| self.full_reference(value)
|| self.env_end(value, 0) == Some(value.len())
{
return Ok(());
}
if hits.len() >= 100_000 {
return Err("Privacy extraction match limit exceeded".into());
}
let mut field = String::new();
let mut previous = None;
for c in key.chars() {
if previous.is_some_and(|v: char| v.is_ascii_lowercase() || v.is_ascii_digit())
&& c.is_ascii_uppercase()
{
field.push('_');
}
if c == '-' || c == ' ' {
field.push('_');
} else {
field.extend(c.to_lowercase());
}
previous = Some(c);
}
hits.push(Hit { start, end, field });
Ok(())
}
fn mask_markdown_markers(&self, s: &str) -> Option<String> {
let delimiters: Vec<_> = self.code_delimiter.find_iter(s).collect();
// Index equal-length runs once so unmatched inline delimiters stay linear-time.
let mut next = vec![None; delimiters.len()];
let mut following = HashMap::new();
for (i, delimiter) in delimiters.iter().enumerate().rev() {
let key = (delimiter.as_str().as_bytes()[0], delimiter.len());
next[i] = following.insert(key, i);
}
let mut masked = None;
let mut cursor = 0;
let mut i = 0;
while i < delimiters.len() {
let open = delimiters[i];
let marker = open.as_str().as_bytes()[0];
let fenced = open.len() >= 3
&& fence_line_start(s, open.start())
&& (marker != b'`' || !s[open.end()..].split('\n').next().unwrap().contains('`'));
let close = if fenced {
(i + 1..delimiters.len()).find(|&j| {
let m = delimiters[j];
m.as_str().as_bytes()[0] == marker
&& m.len() >= open.len()
&& fence_line_start(s, m.start())
&& s[m.end()..].split('\n').next().unwrap().trim().is_empty()
})
} else if marker == b'`' {
next[i]
} else {
None
};
if close.is_none() && !fenced {
i += 1;
continue;
}
self.mask_bold_markers(s, cursor, open.start(), &mut masked);
let bytes = masked.get_or_insert_with(|| s.as_bytes().to_vec());
bytes[open.start()..open.end()].fill(b' ');
cursor = if let Some(j) = close {
let end = delimiters[j];
bytes[end.start()..end.end()].fill(b' ');
i = j + 1;
end.end()
} else {
i = delimiters.len();
s.len()
};
}
self.mask_bold_markers(s, cursor, s.len(), &mut masked);
masked.map(|bytes| String::from_utf8(bytes).expect("ASCII replacement preserves UTF-8"))
}
fn mask_bold_markers(&self, s: &str, start: usize, end: usize, masked: &mut Option<Vec<u8>>) {
// Emphasis inside code is literal; only scan the gaps between code regions.
for cap in self.bold.captures_iter(&s[start..end]) {
let text = cap.name("text").unwrap();
let bytes = masked.get_or_insert_with(|| s.as_bytes().to_vec());
bytes[start + text.start() - 2..start + text.start()].fill(b' ');
bytes[start + text.end()..start + text.end() + 2].fill(b' ');
}
}
fn value_span(&self, s: &str, mut start: usize) -> (usize, usize) {
if s[start..].starts_with("${") {
if let Some(end) = self.env_end(s, start) {
return (start, end);
}
}
let mut end = start;
if let Some(quote @ ('\'' | '"')) = char_at(s, start) {
start += 1;
end = start;
while let Some(c) = char_at(s, end) {
if c == '\\' {
end = advance(s, end, 2);
} else if c == quote {
break;
} else {
end += c.len_utf8();
}
}
} else {
while let Some(c) = char_at(s, end) {
if space(c) || ",;,;\"'<>}{]".contains(c) {
break;
}
if c == '[' {
match self.bracket.find_at(s, end) {
Some(m) if m.start() == end => end = m.end(),
_ => break,
}
} else {
end += c.len_utf8();
}
}
}
(start, end)
}
fn shell_span(&self, s: &str, start: usize) -> (usize, usize) {
let encoded = s[start..].starts_with("\\\"");
let mut quote = if encoded { Some('"') } else { None };
let width = if encoded { 2 } else { 1 };
let mut end = start + if encoded { 2 } else { 0 };
let mut first_quote_end = None;
while let Some(c) = char_at(s, end) {
if encoded && c == '\\' && quote != Some('\'') {
let mut run = end + 1;
while char_at(s, run) == Some('\\') {
run += 1;
}
if run == end + 1 && char_at(s, run) == Some('"') {
if quote.is_some() {
quote = None;
if first_quote_end.is_none() {
first_quote_end = Some(end);
}
} else {
quote = Some('"');
}
end = run + 1;
} else {
end = advance(s, run, 1);
}
} else if c == '\\' && quote != Some('\'') {
end = advance(s, end, 2);
} else if let Some(q) = quote {
if c == q && (!encoded || q == '\'') {
quote = None;
if first_quote_end.is_none() {
first_quote_end = Some(end);
}
}
end += c.len_utf8();
} else if c == '\'' || c == '"' {
if encoded && c == '"' {
break;
}
quote = Some(c);
end += 1;
} else if space(c) || ";&|()<>".contains(c) {
break;
} else {
end += c.len_utf8();
}
}
if start < end
&& (encoded || matches!(char_at(s, start), Some('\'' | '"')))
&& first_quote_end == end.checked_sub(width)
{
(start + width, end - width)
} else {
(start, end)
}
}
fn scheme_end(&self, s: &str, pos: usize, end: usize) -> Option<usize> {
if pos > 0 && self.word(s[..pos].chars().next_back()?) {
return None;
}
self.scheme
.find_at(&s[..end], pos)
.filter(|m| m.start() == pos)
.map(|m| m.end())
}
/// Return disjoint credential spans in Unicode character coordinates.
pub fn extract(
&self,
s: &str,
field_name: Option<&str>,
) -> Result<Vec<(usize, usize, String)>, String> {
if s.len() > 4 * 1024 * 1024 && s.chars().count() > 4 * 1024 * 1024 {
return Err("Privacy extraction text limit exceeded".into());
}
let mut hits = Vec::new();
if let Some(field) = field_name.filter(|v| self.is_sensitive_key(v)) {
let mut start = 0;
if matches!(
normalize_key(&field.to_lowercase()).as_str(),
"authorization" | "proxyauthorization"
) {
start = self.scheme_end(s, 0, s.len()).unwrap_or(0);
}
self.add(&mut hits, s, start, s.len(), field, true)?;
return Ok(to_char_offsets(s, hits));
}
// Replace Markdown delimiters with whitespace, retaining original offsets and values.
let source = s;
let scan_text = self.mask_markdown_markers(s);
let s = scan_text.as_deref().unwrap_or(s);
if s.contains(['=', ':', ':']) {
let mut cursor = 0;
let mut covered = 0;
while cursor < s.len() {
let Some(cap) = self.assignment.captures_at(s, cursor) else {
break;
};
let m = cap.get(0).unwrap();
if !self.boundary(s, m.start()) {
cursor = advance(s, m.start(), 1);
continue;
}
cursor = m.end();
let key = cap.name("key").unwrap().as_str();
if m.start() < covered || !self.is_sensitive_key(key) {
continue;
}
let start = m.end();
let auth_start = start + usize::from(matches!(char_at(s, start), Some('\'' | '"')));
let start = if matches!(
normalize_key(&key.to_lowercase()).as_str(),
"authorization" | "proxyauthorization"
) {
self.scheme_end(s, auth_start, s.len()).unwrap_or(start)
} else {
start
};
let (start, end) = self.value_span(s, start);
self.add(&mut hits, source, start, end, key, false)?;
covered = end;
}
}
if s.contains("--") || s.contains("-u") || s.contains("-U") {
let mut cursor = 0;
let mut covered = 0;
while cursor < s.len() {
let Some(cap) = self.cli.captures_at(s, cursor) else {
break;
};
let m = cap.get(0).unwrap();
if !self.boundary(s, m.start()) {
cursor = advance(s, m.start(), 1);
continue;
}
cursor = m.end();
if cap.name("short").is_some() && m.end() == m.start() + 2 {
if char_at(s, m.end()).map_or(true, |c| space(c) || c == '=') {
continue;
}
}
if m.start() < covered {
continue;
}
let mut key = cap.name("key").map_or("user", |v| v.as_str());
let normalized = normalize_key(&key.to_lowercase());
let combined = normalized == "user" || normalized == "proxyuser";
if normalized == "oauth2bearer" {
key = "access_token";
}
if !combined && !self.is_sensitive_key(key) {
continue;
}
let mut start = m.end();
loop {
let count = if s[start..].starts_with("\\\r\n") {
3
} else if s[start..].starts_with("\\\n") {
2
} else {
break;
};
start += count;
while matches!(char_at(s, start), Some(' ' | '\t')) {
start += 1;
}
}
if s[start..].starts_with("--") {
continue;
}
let (mut start, end) = self.shell_span(s, start);
if end > start && (!combined || s[start..end].contains(':')) {
covered = end;
if normalized == "authorization" || normalized == "proxyauthorization" {
start = self.scheme_end(s, start, end).unwrap_or(start);
}
self.add(
&mut hits,
source,
start,
end,
if combined { "credential" } else { key },
false,
)?;
}
}
}
if self.cfg.token_hints.iter().any(|h| s.contains(h)) {
let mut cursor = 0;
while cursor < s.len() {
let Some(m) = self.token.find_at(s, cursor) else {
break;
};
if !self.boundary(s, m.start()) {
cursor = advance(s, m.start(), 1);
continue;
}
cursor = m.end();
if (m.as_str().starts_with("AKIA") || m.as_str().starts_with("ASIA"))
&& char_at(s, m.end()).is_some_and(token_char)
{
continue;
}
self.add(&mut hits, source, m.start(), m.end(), "credential", false)?;
}
}
if s.contains(['b', 'B']) {
let mut cursor = 0;
while cursor < s.len() {
let Some(cap) = self.auth.captures_at(s, cursor) else {
break;
};
let m = cap.get(0).unwrap();
if m.start() > 0 && self.word(s[..m.start()].chars().next_back().unwrap()) {
cursor = advance(s, m.start(), 1);
continue;
}
cursor = m.end();
let value = cap.name("value").unwrap();
self.add(
&mut hits,
source,
value.start(),
value.end(),
"authorization",
false,
)?;
}
}
let refs: Vec<_> = self
.reference
.find_iter(s)
.map(|m| (m.start(), m.end()))
.collect();
let mut r = 0;
let mut merged: Vec<Hit> = Vec::new();
hits.sort_by_key(|h| h.start);
for h in hits {
while r < refs.len() && refs[r].1 <= h.start {
r += 1;
}
if r < refs.len() && refs[r].0 <= h.start && h.end <= refs[r].1 {
continue;
}
if let Some(last) = merged.last_mut() {
if h.start < last.end {
last.end = last.end.max(h.end);
continue;
}
}
merged.push(h);
}
Ok(to_char_offsets(source, merged))
}
}
fn fence_line_start(s: &str, pos: usize) -> bool {
let mut prefix = s[..pos].bytes().rev();
for _ in 0..3 {
match prefix.next() {
None | Some(b'\n') => return true,
Some(b' ') => (),
_ => return false,
}
}
matches!(prefix.next(), None | Some(b'\n'))
}
fn normalize_key(s: &str) -> String {
s.chars()
.filter(|c| !matches!(c, '-' | '_' | ' '))
.collect()
}
fn space(c: char) -> bool {
c.is_whitespace() || matches!(c, '\u{1c}'..='\u{1f}')
}
fn token_char(c: char) -> bool {
c.is_ascii_alphanumeric() || "_./+=~-".contains(c)
}
fn char_at(s: &str, p: usize) -> Option<char> {
s.get(p..)?.chars().next()
}
fn advance(s: &str, mut p: usize, n: usize) -> usize {
for _ in 0..n {
if let Some(c) = char_at(s, p) {
p += c.len_utf8();
}
}
p
}
fn to_char_offsets(s: &str, hits: Vec<Hit>) -> Vec<(usize, usize, String)> {
if s.is_ascii() {
return hits
.into_iter()
.map(|h| (h.start, h.end, h.field))
.collect();
}
let mut byte = 0;
let mut count = 0;
let mut out = Vec::with_capacity(hits.len());
for h in hits {
count += s[byte..h.start].chars().count();
let start = count;
count += s[h.start..h.end].chars().count();
byte = h.end;
out.push((start, count, h.field));
}
out
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn basic_requires_authentication_context() {
let extractor = PrivacyExtractor::new().unwrap();
let text = "Basic usage; Authorization: Basic demo-secret";
let chars: Vec<_> = text.chars().collect();
let values: Vec<String> = extractor
.extract(text, None)
.unwrap()
.iter()
.map(|(start, end, _)| chars[*start..*end].iter().collect())
.collect();
assert_eq!(values, ["demo-secret"]);
assert_eq!(
extractor
.extract("Basic demo-secret", Some("Authorization"))
.unwrap(),
vec![(6, 17, "authorization".to_owned())]
);
}
#[test]
fn unicode_offsets_and_overlaps_cover_complete_credentials() {
let extractor = PrivacyExtractor::new().unwrap();
let text = "前缀 password='中文密码' Authorization: Bearer sk-abcdefghijklmnopqrstuv";
let hits = extractor.extract(text, None).unwrap();
let chars: Vec<_> = text.chars().collect();
let values: Vec<String> = hits
.iter()
.map(|(s, e, _)| chars[*s..*e].iter().collect())
.collect();
assert_eq!(values, ["中文密码", "sk-abcdefghijklmnopqrstuv"]);
}
#[test]
fn references_and_environment_placeholders_are_not_new_credentials() {
let extractor = PrivacyExtractor::new().unwrap();
let text = "password=${PASSWORD} {{ov_privacy:alice:viking://~/privacy/session/sk-abcdefghijklmnopqrstuv#/values/pat}}";
assert!(extractor.extract(text, None).unwrap().is_empty());
for value in ["${PASSWORD}", "$PASSWORD中", "$PASSWORD²", "$PASSWORD¼"] {
assert!(extractor
.extract(value, Some("password"))
.unwrap()
.is_empty());
}
// A combining mark ends the environment name and leaves a literal value.
assert_eq!(
extractor
.extract("$PASSWORD\u{0345}", Some("password"))
.unwrap(),
vec![(0, 10, "password".to_owned())]
);
}
#[test]
fn markdown_delimiters_preserve_value_boundaries_and_literal_symbols() {
let extractor = PrivacyExtractor::new().unwrap();
for (text, expected) in [
("**API_KEY=secret-value**", "secret-value"),
("1. **API_KEY=secret-value**.", "secret-value"),
("password=**secret-value**", "secret-value"),
("**Authorization: Bearer secret-value**", "secret-value"),
("**sk-abcdefghijklmnopqrstuv**", "sk-abcdefghijklmnopqrstuv"),
("password=secret**", "secret**"),
("password=ab**cd", "ab**cd"),
("password=***", "***"),
("**password='secret**'**", "secret**"),
("password='**secret**'", "**secret**"),
("`API_KEY=secret-value`", "secret-value"),
("password=`secret-value`", "secret-value"),
("``API_KEY=secret-value``", "secret-value"),
("**`API_KEY=secret-value`**", "secret-value"),
("**API_KEY=`secret-value`**", "secret-value"),
("`password=**secret**`", "**secret**"),
("``password=has`tick``", "has`tick"),
("````password=has```ticks````", "has```ticks"),
("`Authorization: Bearer secret-value`", "secret-value"),
("`--api-key secret-value`", "secret-value"),
("``--password='secret`**'``", "secret`**"),
("password='literal`tick`**'", "literal`tick`**"),
("password=unpaired`", "unpaired`"),
("`unclosed password=secret-value", "secret-value"),
("`password='**secret**'`", "**secret**"),
("```\npassword=**secret**\n```", "**secret**"),
("```sh\npassword=**secret**\n````", "**secret**"),
("~~~sh\npassword=**secret**\n~~~~", "**secret**"),
("```sh\npassword=**secret**", "**secret**"),
("~~~sh\npassword=**secret**", "**secret**"),
("**heading** password=secret**", "secret**"),
("中文 **password=密码**", "密码"),
("中文 `password=密码`。", "密码"),
] {
let chars: Vec<_> = text.chars().collect();
let values: Vec<String> = extractor
.extract(text, None)
.unwrap()
.iter()
.map(|(s, e, _)| chars[*s..*e].iter().collect())
.collect();
assert_eq!(values, [expected], "{text}");
}
assert_eq!(
extractor.extract("**secret**", Some("password")).unwrap(),
vec![(0, 10, "password".to_owned())]
);
let text = "`password=one` **password=two** ``password=three`tick`` password='four`**'";
let chars: Vec<_> = text.chars().collect();
let values: Vec<String> = extractor
.extract(text, None)
.unwrap()
.iter()
.map(|(s, e, _)| chars[*s..*e].iter().collect())
.collect();
assert_eq!(values, ["one", "two", "three`tick", "four`**"]);
}
#[test]
fn unicode_categories_preserve_credential_boundaries() {
let extractor = PrivacyExtractor::new().unwrap();
// Letters, numbers and underscores continue identifiers.
for prefix in ['A', '9', '_', '中', '²', '¼'] {
for suffix in ["sk-abcdefghijklmnopqrstuv", "Bearer secret-value"] {
let text = format!("{prefix}{suffix}");
assert!(extractor.extract(&text, None).unwrap().is_empty(), "{text}");
}
}
// Combining marks, other connectors and format characters form boundaries.
for prefix in ['\u{0345}', '\u{0363}', '\u{203f}', '\u{200c}', '🙂'] {
let text = format!("{prefix}password=secret-value");
assert_eq!(
extractor.extract(&text, None).unwrap(),
vec![(10, 22, "password".to_owned())]
);
}
}
}