Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
fix(hooks): keep hook events whose payload has a lone surrogate escape
Cayan7cbb09d6608b
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex da1fca12..1df24c59 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -12,6 +12,11 @@ version in [`docs/changelog/`](docs/changelog/):## [Unreleased]### Fixed+- Kept native hook events whose JSON payload contains half of a UTF-16+ surrogate pair as a `\u` escape, such as tool output cut in the middle of+ an emoji. The hook could not parse the payload and dropped the whole event+ with "nothing was captured"; the unpaired escape now becomes U+FFFD. The+ shell and PowerShell hook scripts still post such a payload unchanged.- Added exact hover timestamps to `/web` page metadata, made the OKF upgradedialog keyboard accessible, and added a labelled main-navigation landmarkwith current-page state and narrow-screen wrapping.diff --git a/crates/ai-memory-cli/src/commands/hook.rs b/crates/ai-memory-cli/src/commands/hook.rsindex c68e6117..2fe2458f 100644--- a/crates/ai-memory-cli/src/commands/hook.rs+++ b/crates/ai-memory-cli/src/commands/hook.rs@@ -1482,8 +1482,54 @@ fn parse_hook_payload(mut payload: String) -> serde_json::Result<(String, serde_if payload.starts_with('\u{feff}') {payload.drain(..'\u{feff}'.len_utf8());}- let json = serde_json::from_str(&payload)?;- Ok((payload, json))+ match serde_json::from_str(&payload) {+ Ok(json) => Ok((payload, json)),+ // The repaired text becomes the spooled body: the server parses it+ // with `serde_json` too and would reject the original.+ Err(error) => {+ let repaired = replace_lone_surrogate_escapes(&payload).ok_or(error)?;+ let json = serde_json::from_str(&repaired)?;+ Ok((repaired, json))+ }+ }+}++/// Replace each `\u` escape that names half of a UTF-16 surrogate pair without+/// its other half with U+FFFD. Harnesses that cut tool output mid-emoji send+/// such escapes, and `serde_json` rejects the whole payload for one of them.+/// An escaped backslash (`\\ud83d`) is text and stays as it is. Returns `None`+/// when nothing was replaced.+fn replace_lone_surrogate_escapes(text: &str) -> Option<String> {+ let code_unit_at = |at: usize| {+ let digits = text.as_bytes().get(at..at + 6)?.strip_prefix(b"\\u")?;+ u16::from_str_radix(std::str::from_utf8(digits).ok()?, 16).ok()+ };+ let (mut out, mut copied, mut at) = (String::new(), 0, 0);+ // `get` ends the scan instead of panicking when invalid JSON puts a+ // backslash before a multibyte character or at the very end.+ while let Some(offset) = text.get(at..).and_then(|rest| rest.find('\\')) {+ at += offset;+ match code_unit_at(at) {+ // `\\`, `\n`, `\"` and the like: skip the escaped character too.+ None => at += 2,+ Some(0xD800..=0xDBFF) if matches!(code_unit_at(at + 6), Some(0xDC00..=0xDFFF)) => {+ at += 12;+ }+ Some(0xD800..=0xDFFF) => {+ out.push_str(&text[copied..at]);+ out.push(char::REPLACEMENT_CHARACTER);+ at += 6;+ copied = at;+ }+ Some(_) => at += 6,+ }+ }+ // A replacement always moves `copied` past its escape.+ if copied == 0 {+ return None;+ }+ out.push_str(&text[copied..]);+ Some(out)}fn hook_context(agent: &str, raw: &serde_json::Value) -> (Option<String>, Option<String>) {diff --git a/crates/ai-memory-cli/tests/suite/hook_payload.rs b/crates/ai-memory-cli/tests/suite/hook_payload.rsindex 6d67c13b..dc2bce24 100644--- a/crates/ai-memory-cli/tests/suite/hook_payload.rs+++ b/crates/ai-memory-cli/tests/suite/hook_payload.rs@@ -457,3 +457,84 @@ fn codex_capture_off_does_not_capture_assistant_message() {// Unrelated fields survive so the Codex Stop event is still ingested.assert!(body.contains("codex-off"), "session_id was dropped: {body}");}++// --- Lone UTF-16 surrogate escapes -----------------------------------------+//+// A harness that cuts tool output in the middle of an emoji can send half of a+// `\uD83D\uDE00` pair. `serde_json` rejects that escape, and so does the+// server's JSON extractor, so the hook replaces only the unpaired half with+// U+FFFD instead of dropping the whole event.++fn spooled_stdout(tmp: &Path, payload: &[u8]) -> String {+ let output = run_hook_event(tmp, "post-tool-use", payload);+ assert!(output.status.success());+ assert!(+ output.stderr.is_empty(),+ "stderr: {}",+ String::from_utf8_lossy(&output.stderr)+ );+ let body: serde_json::Value =+ serde_json::from_str(&spooled_body(tmp)).expect("spooled body is JSON");+ body["tool_response"]["stdout"]+ .as_str()+ .expect("tool_response.stdout")+ .to_owned()+}++#[test]+fn lone_leading_surrogate_escape_is_spooled_as_replacement_character() {+ let tmp = tempfile::tempdir().expect("tempdir");+ let payload = br#"{"session_id":"lone-lead","cwd":"/tmp","tool_name":"Bash","tool_input":{"command":"echo"},"tool_response":{"stdout":"ok \ud83d\n... (truncated)"}}"#;+ assert_eq!(+ spooled_stdout(tmp.path(), payload),+ "ok \u{FFFD}\n... (truncated)"+ );+}++#[test]+fn lone_trailing_surrogate_escape_is_spooled_as_replacement_character() {+ let tmp = tempfile::tempdir().expect("tempdir");+ let payload = br#"{"session_id":"lone-trail","cwd":"/tmp","tool_name":"Bash","tool_input":{"command":"echo"},"tool_response":{"stdout":"\ude00 tail"}}"#;+ assert_eq!(spooled_stdout(tmp.path(), payload), "\u{FFFD} tail");+}++#[test]+fn surrogate_repair_keeps_pairs_and_escaped_backslashes() {+ // A lead followed by a full pair: only the first lead is unpaired. The+ // `\\ud83d` is an escaped backslash followed by text, not an escape.+ let tmp = tempfile::tempdir().expect("tempdir");+ let payload = br#"{"session_id":"mixed","cwd":"/tmp","tool_name":"Bash","tool_input":{"command":"echo"},"tool_response":{"stdout":"\ud83d\ud83d\ude00 \\ud83d \\\ude00"}}"#;+ assert_eq!(+ spooled_stdout(tmp.path(), payload),+ "\u{FFFD}\u{1F600} \\ud83d \\\u{FFFD}"+ );+}++#[test]+fn payload_with_a_full_surrogate_pair_keeps_its_exact_bytes() {+ let tmp = tempfile::tempdir().expect("tempdir");+ let payload = br#"{"session_id":"pair","cwd":"/tmp","tool_name":"Bash","tool_input":{"command":"echo"},"tool_response":{"stdout":"ok \ud83d\ude00"}}"#;+ let output = run_hook_event(tmp.path(), "post-tool-use", payload);+ assert!(output.status.success());+ assert_eq!(spooled_body(tmp.path()).as_bytes(), payload);+}++#[test]+fn malformed_payload_with_a_lone_surrogate_still_warns_without_spooling() {+ // Invalid JSON stays invalid after the repair; a backslash before a+ // multibyte character or at the end must not crash the hook.+ for payload in [+ r#"{"x":"\ud83d \é"#.as_bytes(),+ r#"{"x":"\ud83d \"#.as_bytes(),+ ] {+ let tmp = tempfile::tempdir().expect("tempdir");+ let output = run_hook(tmp.path(), payload);+ assert!(output.status.success());+ assert_eq!(output.stdout, b"{}\n");+ assert_eq!(+ String::from_utf8(output.stderr).expect("stderr utf8"),+ "ai-memory hook warning: could not parse event payload as JSON; nothing was captured\n"+ );+ assert!(!tmp.path().join("hook-spool").exists());+ }+}