Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

fix(sanitize): redact JSON secrets and Basic auth, strip control sequences

Gabe Tobias8e42800a6156
diff --git a/.gitleaks.toml b/.gitleaks.toml
index 16457c01..9072ac3f 100644
--- a/.gitleaks.toml
+++ b/.gitleaks.toml
@@ -55,6 +55,12 @@ regexes = [
'''123456:ABC-DEF1234ghIkl-zyx57W2v1u123ew11''',
'''ABC-DEF1234ghIkl-zyx57W2v1u123ew11''',
'''Bearer abcdef0123456789ABCDEF0123456789''',
+ # Fixtures for the JSON-quoted value, scheme-prefixed header and
+ # unprefixed-assignment tests: a JSON `api_key`, a `client-token`, an
+ # `authorization: Token`, an Azure `AccountKey=` and an npm `_authToken=`.
+ # `generic-api-key` extracts the value alone, so the FAKE marker lives
+ # inside the value rather than in the key.
+ '''FAKEfake0123456789[A-Za-z0-9]*={0,2}''',
'''xoxb-1234567890-abcdefghij''',
# ---- Canary string used by the e2e test in tests/e2e/ ----
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 08a74e62..3f6912aa 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
+### Fixed
+- The privacy strip now redacts secrets written in JSON. The quote before a
+ value put it outside the value character class, so `{"db_password":"..."}`
+ was stored verbatim while the identical YAML form was redacted, and JSON is
+ the shape most captured tool payloads arrive in. The same rule now also
+ accepts an auth scheme word before the value, so
+ `Authorization: Basic <base64>` (which carries `user:password`) is redacted
+ like the `Bearer` form already was, and covers two unprefixed names the
+ generic env rule missed: Azure `AccountKey=` and npm `_authToken=`.
+- Terminal escape sequences, NUL and bidi override characters are stripped
+ from captured text instead of being stored. Page bodies and observations are
+ replayed to a terminal by `ai-memory read-page` and `ai-memory search`, where
+ an escape rewrites the screen or the window title and a bidi override
+ reverses what the reader sees; a NUL additionally made the markdown file
+ binary, costing it `grep` and git diffs. Tabs, newlines and carriage returns
+ are kept.
+
## [2.3.2] - 2026-09-20
### Changed
diff --git a/crates/ai-memory-core/src/sanitize.rs b/crates/ai-memory-core/src/sanitize.rs
index 739fa15c..6ea3540c 100644
--- a/crates/ai-memory-core/src/sanitize.rs
+++ b/crates/ai-memory-core/src/sanitize.rs
@@ -26,6 +26,10 @@
//! `[sanitize].allowlist` — the allowlist is checked *per match*, so a
//! pattern still runs but an allowlisted span survives unchanged.
//!
+//! Terminal escape sequences, NUL and bidi overrides are stripped rather
+//! than redacted: they are not secrets, but stored text is replayed to a
+//! terminal by the CLI, and a NUL makes the markdown file binary.
+//!
//! ## What we deliberately do not catch
//!
//! Standalone high-entropy strings (e.g. a 32-char random hex) cannot
@@ -45,6 +49,20 @@ use crate::NewObservation;
/// boundary above 16 KiB.
pub const OBSERVATION_BODY_MAX_BYTES: usize = 16 * 1024;
+/// ANSI/VT escape sequences: CSI (`ESC [ ... final`), OSC (`ESC ] ... BEL` or
+/// `ESC \\`), and the two-character forms. Removed whole, so a colour code
+/// does not leave `[31m` behind as text.
+const ESCAPE_SEQUENCES: &str =
+ r"\x1b\[[0-9;?]*[ -/]*[@-~]|\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)|\x1b[@-Z\\-_]";
+
+/// C0 and C1 controls except tab, newline and carriage return, DEL, and the
+/// bidi overrides that let stored text render as something else.
+const fn is_stripped_control(c: char) -> bool {
+ matches!(c,
+ '\u{0}'..='\u{8}' | '\u{b}' | '\u{c}' | '\u{e}'..='\u{1f}'
+ | '\u{7f}'..='\u{9f}' | '\u{202a}'..='\u{202e}' | '\u{2066}'..='\u{2069}')
+}
+
/// Compile-time list of redaction patterns. Order is intentional:
/// more-specific patterns first. False positives are acceptable —
/// better to redact a stray hash than to leak a credential.
@@ -145,20 +163,20 @@ const BUILTIN_PATTERNS: &[(&str, &str)] = &[
// protects an already-redacted value: `[REDACTED]` starts with `[`,
// which the value character class excludes outright.
(
- r#"(?i)\b[A-Za-z0-9-]*(?:authentication|authorization|credentials?|password|passwd|apikey|[a-z0-9]*(?:api|auth|access|secret|security|private|session|refresh|client|consumer|subscription|app|bearer)-(?:key|token))\s*:\s*[A-Za-z0-9._~+/=-]{8,}"#,
+ r#"(?i)\b[A-Za-z0-9_-]*(?:authentication|authorization|credentials?|password|passwd|apikey|accountkey|authtoken|[a-z0-9]*(?:api|auth|access|secret|security|private|session|refresh|client|consumer|subscription|app|bearer)-(?:key|token))"?\s*[=:]\s*(?:(?:basic|bearer|digest|token|apikey)\s+)?"?[A-Za-z0-9._~+/=-]{8,}"#,
"auth_header",
),
// Provider-specific env-var assignments (kept explicit for clarity
// and so that bare `OPENAI_API_KEY=anything-at-all` still triggers
// even without `sk-` shape).
(
- r#"(?i)(ANTHROPIC_API_KEY|OPENAI_API_KEY|OPENROUTER_API_KEY|VOYAGE_API_KEY|MISTRAL_API_KEY|GROQ_API_KEY|HF_TOKEN|HUGGINGFACE_TOKEN|AWS_(SECRET_)?ACCESS_KEY[A-Z_]*|GITHUB_TOKEN|GH_TOKEN|GITLAB_TOKEN|GOOGLE_API_KEY|GEMINI_API_KEY|OLLAMA_API_KEY)\s*[=:]\s*\S+"#,
+ r#"(?i)(ANTHROPIC_API_KEY|OPENAI_API_KEY|OPENROUTER_API_KEY|VOYAGE_API_KEY|MISTRAL_API_KEY|GROQ_API_KEY|HF_TOKEN|HUGGINGFACE_TOKEN|AWS_(SECRET_)?ACCESS_KEY[A-Z_]*|GITHUB_TOKEN|GH_TOKEN|GITLAB_TOKEN|GOOGLE_API_KEY|GEMINI_API_KEY|OLLAMA_API_KEY)"?\s*[=:]\s*\S+"#,
"env_secret",
),
// Generic env-var catch-all: any *_KEY / *_TOKEN / *_SECRET /
// *_PASSWORD / *_CREDENTIAL[S] / *_PRIVATE_KEY assignment.
(
- r#"(?i)\b[A-Z][A-Z0-9_]*_(KEY|TOKEN|SECRET|PASSWORD|PASSWD|CREDENTIAL|CREDENTIALS|PRIVATE_KEY)\s*[=:]\s*\S+"#,
+ r#"(?i)\b[A-Z][A-Z0-9_]*_(KEY|TOKEN|SECRET|PASSWORD|PASSWD|CREDENTIAL|CREDENTIALS|PRIVATE_KEY)"?\s*[=:]\s*\S+"#,
"env_secret",
),
// Filesystem paths that commonly contain credentials.
@@ -182,6 +200,7 @@ pub struct Sanitizer {
struct SanitizerInner {
patterns: Vec<(Regex, &'static str)>,
allowlist: Vec<String>,
+ escapes: Regex,
}
impl std::fmt::Debug for Sanitizer {
@@ -227,6 +246,7 @@ impl Sanitizer {
inner: Arc::new(SanitizerInner {
patterns,
allowlist: cfg.allowlist.clone(),
+ escapes: Regex::new(ESCAPE_SEQUENCES)?,
}),
})
}
@@ -243,7 +263,9 @@ impl Sanitizer {
/// allowlist entry, in which case it is left alone.
#[must_use]
pub fn scrub(&self, input: &str) -> String {
- let mut out = input.to_string();
+ // Before the redaction passes: an escape inside a secret would
+ // otherwise split it out of reach of every pattern below.
+ let mut out = self.strip_control(input);
for (re, label) in &self.inner.patterns {
out = re
.replace_all(&out, |caps: &regex::Captures<'_>| {
@@ -259,6 +281,25 @@ impl Sanitizer {
}
out
}
+
+ /// Drop terminal escape sequences and the control characters that have
+ /// no place in a page.
+ ///
+ /// Captured text is replayed to a terminal by `ai-memory read-page` and
+ /// `search`, where an escape rewrites the screen or the window title, and
+ /// a bidi override reverses what the reader sees. A NUL additionally
+ /// makes the markdown file binary, which costs it `grep` and git diffs.
+ fn strip_control(&self, input: &str) -> String {
+ let stripped = self.inner.escapes.replace_all(input, "");
+ if stripped.contains(is_stripped_control) {
+ stripped
+ .chars()
+ .filter(|c| !is_stripped_control(*c))
+ .collect()
+ } else {
+ stripped.into_owned()
+ }
+ }
}
impl Default for Sanitizer {
@@ -880,4 +921,97 @@ mod tests {
"allowlist span should survive; got: {out}"
);
}
+
+ /// JSON is the shape most captured tool payloads arrive in, and the
+ /// quote before the value used to put it outside the value class, so the
+ /// same secret was redacted in YAML and stored verbatim in JSON.
+ #[test]
+ fn scrubs_secret_values_in_json() {
+ for raw in [
+ r#"{"db_password":"correct-horse-battery"}"#,
+ r#"{"password": "hunter2hunter2"}"#,
+ r#""api_key": "FAKEfake0123456789abcdef""#,
+ r#"{"client-token":"FAKEfake0123456789"}"#,
+ ] {
+ let out = s().scrub(raw);
+ assert!(
+ out.contains("[REDACTED:"),
+ "JSON-quoted secret must be redacted: {raw} -> {out}"
+ );
+ }
+ }
+
+ /// `Authorization: Basic <base64>` carries `user:password`. The scheme
+ /// word sits where the value class expected the secret, so the rule that
+ /// names `authorization` never fired; only `Bearer` was caught, and only
+ /// because a separate pattern matches the scheme keyword itself.
+ #[test]
+ fn scrubs_scheme_prefixed_authorization_headers() {
+ for raw in [
+ "Authorization: Basic YWRtaW46c2VjcmV0cGFzc3dvcmQ=",
+ "authorization: Token FAKEfake0123456789",
+ "Proxy-Authorization: Digest cmVhbG09ZXhhbXBsZQ==",
+ ] {
+ let out = s().scrub(raw);
+ assert!(
+ out.contains("[REDACTED:"),
+ "scheme-prefixed credential must be redacted: {raw} -> {out}"
+ );
+ }
+ }
+
+ /// Azure storage connection strings and npm `_authToken` name the secret
+ /// without the underscore the generic env rule requires.
+ #[test]
+ fn scrubs_unprefixed_account_and_auth_token_assignments() {
+ for raw in [
+ "DefaultEndpointsProtocol=https;AccountName=x;AccountKey=FAKEfake0123456789==;",
+ "//registry.npmjs.org/:_authToken=npm_FAKEfake0123456789abcdefghij",
+ ] {
+ let out = s().scrub(raw);
+ assert!(
+ out.contains("[REDACTED:"),
+ "assignment must be redacted: {raw} -> {out}"
+ );
+ }
+ }
+
+ /// Page bodies and observations are replayed to a terminal by
+ /// `ai-memory read-page` / `search`, and a NUL makes the markdown file
+ /// binary, so `grep` skips it and git stops diffing it.
+ #[test]
+ fn strips_terminal_control_sequences_and_nul() {
+ let out = s().scrub(
+ "red \u{1b}[31mtext\u{1b}[0m title \u{1b}]0;pwned\u{7}\
+ bidi \u{202e}reversed\u{202c} nul \u{0}end\n\ttabbed",
+ );
+ for bad in ['\u{1b}', '\u{7}', '\u{0}', '\u{202e}', '\u{202c}'] {
+ assert!(
+ !out.contains(bad),
+ "{bad:?} must not survive into a page or a terminal: {out:?}"
+ );
+ }
+ assert!(
+ !out.contains("[31m") && !out.contains("]0;"),
+ "the sequence bodies must go with their escapes: {out:?}"
+ );
+ assert!(
+ out.contains("red text title ") && out.ends_with("end\n\ttabbed"),
+ "text, newlines and tabs must survive: {out:?}"
+ );
+ }
+
+ /// The value floor is what keeps ordinary `key: value` prose intact; the
+ /// new quote and scheme allowances must not erode it.
+ #[test]
+ fn keeps_short_and_non_secret_values() {
+ for raw in [
+ "Access-Control-Allow-Credentials: true",
+ "Idempotency-Key: abc",
+ "note: the password is stored in 1Password",
+ ] {
+ let out = s().scrub(raw);
+ assert_eq!(out, raw, "must be left alone: {raw}");
+ }
+ }
}