Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
fix(sanitize): redact JSON secrets and Basic auth, strip control sequences
Gabe Tobias8e42800a6156
diff --git a/.gitleaks.toml b/.gitleaks.tomlindex 16457c01..9072ac3f 100644--- a/.gitleaks.toml+++ b/.gitleaks.toml@@ -55,6 +55,12 @@ regexes = ['''123456:ABC-DEF1234ghIkl-zyx57W2v1u123ew11''','''ABC-DEF1234ghIkl-zyx57W2v1u123ew11''','''Bearer abcdef0123456789ABCDEF0123456789''',+ # Fixtures for the JSON-quoted value, scheme-prefixed header and+ # unprefixed-assignment tests: a JSON `api_key`, a `client-token`, an+ # `authorization: Token`, an Azure `AccountKey=` and an npm `_authToken=`.+ # `generic-api-key` extracts the value alone, so the FAKE marker lives+ # inside the value rather than in the key.+ '''FAKEfake0123456789[A-Za-z0-9]*={0,2}''','''xoxb-1234567890-abcdefghij''',# ---- Canary string used by the e2e test in tests/e2e/ ----diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 08a74e62..3f6912aa 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -7,6 +7,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0## [Unreleased]+### Fixed+- The privacy strip now redacts secrets written in JSON. The quote before a+ value put it outside the value character class, so `{"db_password":"..."}`+ was stored verbatim while the identical YAML form was redacted, and JSON is+ the shape most captured tool payloads arrive in. The same rule now also+ accepts an auth scheme word before the value, so+ `Authorization: Basic <base64>` (which carries `user:password`) is redacted+ like the `Bearer` form already was, and covers two unprefixed names the+ generic env rule missed: Azure `AccountKey=` and npm `_authToken=`.+- Terminal escape sequences, NUL and bidi override characters are stripped+ from captured text instead of being stored. Page bodies and observations are+ replayed to a terminal by `ai-memory read-page` and `ai-memory search`, where+ an escape rewrites the screen or the window title and a bidi override+ reverses what the reader sees; a NUL additionally made the markdown file+ binary, costing it `grep` and git diffs. Tabs, newlines and carriage returns+ are kept.+## [2.3.2] - 2026-09-20### Changeddiff --git a/crates/ai-memory-core/src/sanitize.rs b/crates/ai-memory-core/src/sanitize.rsindex 739fa15c..6ea3540c 100644--- a/crates/ai-memory-core/src/sanitize.rs+++ b/crates/ai-memory-core/src/sanitize.rs@@ -26,6 +26,10 @@//! `[sanitize].allowlist` — the allowlist is checked *per match*, so a//! pattern still runs but an allowlisted span survives unchanged.//!+//! Terminal escape sequences, NUL and bidi overrides are stripped rather+//! than redacted: they are not secrets, but stored text is replayed to a+//! terminal by the CLI, and a NUL makes the markdown file binary.+//!//! ## What we deliberately do not catch//!//! Standalone high-entropy strings (e.g. a 32-char random hex) cannot@@ -45,6 +49,20 @@ use crate::NewObservation;/// boundary above 16 KiB.pub const OBSERVATION_BODY_MAX_BYTES: usize = 16 * 1024;+/// ANSI/VT escape sequences: CSI (`ESC [ ... final`), OSC (`ESC ] ... BEL` or+/// `ESC \\`), and the two-character forms. Removed whole, so a colour code+/// does not leave `[31m` behind as text.+const ESCAPE_SEQUENCES: &str =+ r"\x1b\[[0-9;?]*[ -/]*[@-~]|\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)|\x1b[@-Z\\-_]";++/// C0 and C1 controls except tab, newline and carriage return, DEL, and the+/// bidi overrides that let stored text render as something else.+const fn is_stripped_control(c: char) -> bool {+ matches!(c,+ '\u{0}'..='\u{8}' | '\u{b}' | '\u{c}' | '\u{e}'..='\u{1f}'+ | '\u{7f}'..='\u{9f}' | '\u{202a}'..='\u{202e}' | '\u{2066}'..='\u{2069}')+}+/// Compile-time list of redaction patterns. Order is intentional:/// more-specific patterns first. False positives are acceptable —/// better to redact a stray hash than to leak a credential.@@ -145,20 +163,20 @@ const BUILTIN_PATTERNS: &[(&str, &str)] = &[// protects an already-redacted value: `[REDACTED]` starts with `[`,// which the value character class excludes outright.(- r#"(?i)\b[A-Za-z0-9-]*(?:authentication|authorization|credentials?|password|passwd|apikey|[a-z0-9]*(?:api|auth|access|secret|security|private|session|refresh|client|consumer|subscription|app|bearer)-(?:key|token))\s*:\s*[A-Za-z0-9._~+/=-]{8,}"#,+ r#"(?i)\b[A-Za-z0-9_-]*(?:authentication|authorization|credentials?|password|passwd|apikey|accountkey|authtoken|[a-z0-9]*(?:api|auth|access|secret|security|private|session|refresh|client|consumer|subscription|app|bearer)-(?:key|token))"?\s*[=:]\s*(?:(?:basic|bearer|digest|token|apikey)\s+)?"?[A-Za-z0-9._~+/=-]{8,}"#,"auth_header",),// Provider-specific env-var assignments (kept explicit for clarity// and so that bare `OPENAI_API_KEY=anything-at-all` still triggers// even without `sk-` shape).(- r#"(?i)(ANTHROPIC_API_KEY|OPENAI_API_KEY|OPENROUTER_API_KEY|VOYAGE_API_KEY|MISTRAL_API_KEY|GROQ_API_KEY|HF_TOKEN|HUGGINGFACE_TOKEN|AWS_(SECRET_)?ACCESS_KEY[A-Z_]*|GITHUB_TOKEN|GH_TOKEN|GITLAB_TOKEN|GOOGLE_API_KEY|GEMINI_API_KEY|OLLAMA_API_KEY)\s*[=:]\s*\S+"#,+ r#"(?i)(ANTHROPIC_API_KEY|OPENAI_API_KEY|OPENROUTER_API_KEY|VOYAGE_API_KEY|MISTRAL_API_KEY|GROQ_API_KEY|HF_TOKEN|HUGGINGFACE_TOKEN|AWS_(SECRET_)?ACCESS_KEY[A-Z_]*|GITHUB_TOKEN|GH_TOKEN|GITLAB_TOKEN|GOOGLE_API_KEY|GEMINI_API_KEY|OLLAMA_API_KEY)"?\s*[=:]\s*\S+"#,"env_secret",),// Generic env-var catch-all: any *_KEY / *_TOKEN / *_SECRET /// *_PASSWORD / *_CREDENTIAL[S] / *_PRIVATE_KEY assignment.(- r#"(?i)\b[A-Z][A-Z0-9_]*_(KEY|TOKEN|SECRET|PASSWORD|PASSWD|CREDENTIAL|CREDENTIALS|PRIVATE_KEY)\s*[=:]\s*\S+"#,+ r#"(?i)\b[A-Z][A-Z0-9_]*_(KEY|TOKEN|SECRET|PASSWORD|PASSWD|CREDENTIAL|CREDENTIALS|PRIVATE_KEY)"?\s*[=:]\s*\S+"#,"env_secret",),// Filesystem paths that commonly contain credentials.@@ -182,6 +200,7 @@ pub struct Sanitizer {struct SanitizerInner {patterns: Vec<(Regex, &'static str)>,allowlist: Vec<String>,+ escapes: Regex,}impl std::fmt::Debug for Sanitizer {@@ -227,6 +246,7 @@ impl Sanitizer {inner: Arc::new(SanitizerInner {patterns,allowlist: cfg.allowlist.clone(),+ escapes: Regex::new(ESCAPE_SEQUENCES)?,}),})}@@ -243,7 +263,9 @@ impl Sanitizer {/// allowlist entry, in which case it is left alone.#[must_use]pub fn scrub(&self, input: &str) -> String {- let mut out = input.to_string();+ // Before the redaction passes: an escape inside a secret would+ // otherwise split it out of reach of every pattern below.+ let mut out = self.strip_control(input);for (re, label) in &self.inner.patterns {out = re.replace_all(&out, |caps: ®ex::Captures<'_>| {@@ -259,6 +281,25 @@ impl Sanitizer {}out}++ /// Drop terminal escape sequences and the control characters that have+ /// no place in a page.+ ///+ /// Captured text is replayed to a terminal by `ai-memory read-page` and+ /// `search`, where an escape rewrites the screen or the window title, and+ /// a bidi override reverses what the reader sees. A NUL additionally+ /// makes the markdown file binary, which costs it `grep` and git diffs.+ fn strip_control(&self, input: &str) -> String {+ let stripped = self.inner.escapes.replace_all(input, "");+ if stripped.contains(is_stripped_control) {+ stripped+ .chars()+ .filter(|c| !is_stripped_control(*c))+ .collect()+ } else {+ stripped.into_owned()+ }+ }}impl Default for Sanitizer {@@ -880,4 +921,97 @@ mod tests {"allowlist span should survive; got: {out}");}++ /// JSON is the shape most captured tool payloads arrive in, and the+ /// quote before the value used to put it outside the value class, so the+ /// same secret was redacted in YAML and stored verbatim in JSON.+ #[test]+ fn scrubs_secret_values_in_json() {+ for raw in [+ r#"{"db_password":"correct-horse-battery"}"#,+ r#"{"password": "hunter2hunter2"}"#,+ r#""api_key": "FAKEfake0123456789abcdef""#,+ r#"{"client-token":"FAKEfake0123456789"}"#,+ ] {+ let out = s().scrub(raw);+ assert!(+ out.contains("[REDACTED:"),+ "JSON-quoted secret must be redacted: {raw} -> {out}"+ );+ }+ }++ /// `Authorization: Basic <base64>` carries `user:password`. The scheme+ /// word sits where the value class expected the secret, so the rule that+ /// names `authorization` never fired; only `Bearer` was caught, and only+ /// because a separate pattern matches the scheme keyword itself.+ #[test]+ fn scrubs_scheme_prefixed_authorization_headers() {+ for raw in [+ "Authorization: Basic YWRtaW46c2VjcmV0cGFzc3dvcmQ=",+ "authorization: Token FAKEfake0123456789",+ "Proxy-Authorization: Digest cmVhbG09ZXhhbXBsZQ==",+ ] {+ let out = s().scrub(raw);+ assert!(+ out.contains("[REDACTED:"),+ "scheme-prefixed credential must be redacted: {raw} -> {out}"+ );+ }+ }++ /// Azure storage connection strings and npm `_authToken` name the secret+ /// without the underscore the generic env rule requires.+ #[test]+ fn scrubs_unprefixed_account_and_auth_token_assignments() {+ for raw in [+ "DefaultEndpointsProtocol=https;AccountName=x;AccountKey=FAKEfake0123456789==;",+ "//registry.npmjs.org/:_authToken=npm_FAKEfake0123456789abcdefghij",+ ] {+ let out = s().scrub(raw);+ assert!(+ out.contains("[REDACTED:"),+ "assignment must be redacted: {raw} -> {out}"+ );+ }+ }++ /// Page bodies and observations are replayed to a terminal by+ /// `ai-memory read-page` / `search`, and a NUL makes the markdown file+ /// binary, so `grep` skips it and git stops diffing it.+ #[test]+ fn strips_terminal_control_sequences_and_nul() {+ let out = s().scrub(+ "red \u{1b}[31mtext\u{1b}[0m title \u{1b}]0;pwned\u{7}\+ bidi \u{202e}reversed\u{202c} nul \u{0}end\n\ttabbed",+ );+ for bad in ['\u{1b}', '\u{7}', '\u{0}', '\u{202e}', '\u{202c}'] {+ assert!(+ !out.contains(bad),+ "{bad:?} must not survive into a page or a terminal: {out:?}"+ );+ }+ assert!(+ !out.contains("[31m") && !out.contains("]0;"),+ "the sequence bodies must go with their escapes: {out:?}"+ );+ assert!(+ out.contains("red text title ") && out.ends_with("end\n\ttabbed"),+ "text, newlines and tabs must survive: {out:?}"+ );+ }++ /// The value floor is what keeps ordinary `key: value` prose intact; the+ /// new quote and scheme allowances must not erode it.+ #[test]+ fn keeps_short_and_non_secret_values() {+ for raw in [+ "Access-Control-Allow-Credentials: true",+ "Idempotency-Key: abc",+ "note: the password is stored in 1Password",+ ] {+ let out = s().scrub(raw);+ assert_eq!(out, raw, "must be left alone: {raw}");+ }+ }}