Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

fix(hooks): keep hook events whose payload has a lone surrogate escape

Cayan7cbb09d6608b
diff --git a/CHANGELOG.md b/CHANGELOG.md
index da1fca12..1df24c59 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,6 +12,11 @@ version in [`docs/changelog/`](docs/changelog/):
## [Unreleased]
### Fixed
+- Kept native hook events whose JSON payload contains half of a UTF-16
+ surrogate pair as a `\u` escape, such as tool output cut in the middle of
+ an emoji. The hook could not parse the payload and dropped the whole event
+ with "nothing was captured"; the unpaired escape now becomes U+FFFD. The
+ shell and PowerShell hook scripts still post such a payload unchanged.
- Added exact hover timestamps to `/web` page metadata, made the OKF upgrade
dialog keyboard accessible, and added a labelled main-navigation landmark
with current-page state and narrow-screen wrapping.
diff --git a/crates/ai-memory-cli/src/commands/hook.rs b/crates/ai-memory-cli/src/commands/hook.rs
index c68e6117..2fe2458f 100644
--- a/crates/ai-memory-cli/src/commands/hook.rs
+++ b/crates/ai-memory-cli/src/commands/hook.rs
@@ -1482,8 +1482,54 @@ fn parse_hook_payload(mut payload: String) -> serde_json::Result<(String, serde_
if payload.starts_with('\u{feff}') {
payload.drain(..'\u{feff}'.len_utf8());
}
- let json = serde_json::from_str(&payload)?;
- Ok((payload, json))
+ match serde_json::from_str(&payload) {
+ Ok(json) => Ok((payload, json)),
+ // The repaired text becomes the spooled body: the server parses it
+ // with `serde_json` too and would reject the original.
+ Err(error) => {
+ let repaired = replace_lone_surrogate_escapes(&payload).ok_or(error)?;
+ let json = serde_json::from_str(&repaired)?;
+ Ok((repaired, json))
+ }
+ }
+}
+
+/// Replace each `\u` escape that names half of a UTF-16 surrogate pair without
+/// its other half with U+FFFD. Harnesses that cut tool output mid-emoji send
+/// such escapes, and `serde_json` rejects the whole payload for one of them.
+/// An escaped backslash (`\\ud83d`) is text and stays as it is. Returns `None`
+/// when nothing was replaced.
+fn replace_lone_surrogate_escapes(text: &str) -> Option<String> {
+ let code_unit_at = |at: usize| {
+ let digits = text.as_bytes().get(at..at + 6)?.strip_prefix(b"\\u")?;
+ u16::from_str_radix(std::str::from_utf8(digits).ok()?, 16).ok()
+ };
+ let (mut out, mut copied, mut at) = (String::new(), 0, 0);
+ // `get` ends the scan instead of panicking when invalid JSON puts a
+ // backslash before a multibyte character or at the very end.
+ while let Some(offset) = text.get(at..).and_then(|rest| rest.find('\\')) {
+ at += offset;
+ match code_unit_at(at) {
+ // `\\`, `\n`, `\"` and the like: skip the escaped character too.
+ None => at += 2,
+ Some(0xD800..=0xDBFF) if matches!(code_unit_at(at + 6), Some(0xDC00..=0xDFFF)) => {
+ at += 12;
+ }
+ Some(0xD800..=0xDFFF) => {
+ out.push_str(&text[copied..at]);
+ out.push(char::REPLACEMENT_CHARACTER);
+ at += 6;
+ copied = at;
+ }
+ Some(_) => at += 6,
+ }
+ }
+ // A replacement always moves `copied` past its escape.
+ if copied == 0 {
+ return None;
+ }
+ out.push_str(&text[copied..]);
+ Some(out)
}
fn hook_context(agent: &str, raw: &serde_json::Value) -> (Option<String>, Option<String>) {
diff --git a/crates/ai-memory-cli/tests/suite/hook_payload.rs b/crates/ai-memory-cli/tests/suite/hook_payload.rs
index 6d67c13b..dc2bce24 100644
--- a/crates/ai-memory-cli/tests/suite/hook_payload.rs
+++ b/crates/ai-memory-cli/tests/suite/hook_payload.rs
@@ -457,3 +457,84 @@ fn codex_capture_off_does_not_capture_assistant_message() {
// Unrelated fields survive so the Codex Stop event is still ingested.
assert!(body.contains("codex-off"), "session_id was dropped: {body}");
}
+
+// --- Lone UTF-16 surrogate escapes -----------------------------------------
+//
+// A harness that cuts tool output in the middle of an emoji can send half of a
+// `\uD83D\uDE00` pair. `serde_json` rejects that escape, and so does the
+// server's JSON extractor, so the hook replaces only the unpaired half with
+// U+FFFD instead of dropping the whole event.
+
+fn spooled_stdout(tmp: &Path, payload: &[u8]) -> String {
+ let output = run_hook_event(tmp, "post-tool-use", payload);
+ assert!(output.status.success());
+ assert!(
+ output.stderr.is_empty(),
+ "stderr: {}",
+ String::from_utf8_lossy(&output.stderr)
+ );
+ let body: serde_json::Value =
+ serde_json::from_str(&spooled_body(tmp)).expect("spooled body is JSON");
+ body["tool_response"]["stdout"]
+ .as_str()
+ .expect("tool_response.stdout")
+ .to_owned()
+}
+
+#[test]
+fn lone_leading_surrogate_escape_is_spooled_as_replacement_character() {
+ let tmp = tempfile::tempdir().expect("tempdir");
+ let payload = br#"{"session_id":"lone-lead","cwd":"/tmp","tool_name":"Bash","tool_input":{"command":"echo"},"tool_response":{"stdout":"ok \ud83d\n... (truncated)"}}"#;
+ assert_eq!(
+ spooled_stdout(tmp.path(), payload),
+ "ok \u{FFFD}\n... (truncated)"
+ );
+}
+
+#[test]
+fn lone_trailing_surrogate_escape_is_spooled_as_replacement_character() {
+ let tmp = tempfile::tempdir().expect("tempdir");
+ let payload = br#"{"session_id":"lone-trail","cwd":"/tmp","tool_name":"Bash","tool_input":{"command":"echo"},"tool_response":{"stdout":"\ude00 tail"}}"#;
+ assert_eq!(spooled_stdout(tmp.path(), payload), "\u{FFFD} tail");
+}
+
+#[test]
+fn surrogate_repair_keeps_pairs_and_escaped_backslashes() {
+ // A lead followed by a full pair: only the first lead is unpaired. The
+ // `\\ud83d` is an escaped backslash followed by text, not an escape.
+ let tmp = tempfile::tempdir().expect("tempdir");
+ let payload = br#"{"session_id":"mixed","cwd":"/tmp","tool_name":"Bash","tool_input":{"command":"echo"},"tool_response":{"stdout":"\ud83d\ud83d\ude00 \\ud83d \\\ude00"}}"#;
+ assert_eq!(
+ spooled_stdout(tmp.path(), payload),
+ "\u{FFFD}\u{1F600} \\ud83d \\\u{FFFD}"
+ );
+}
+
+#[test]
+fn payload_with_a_full_surrogate_pair_keeps_its_exact_bytes() {
+ let tmp = tempfile::tempdir().expect("tempdir");
+ let payload = br#"{"session_id":"pair","cwd":"/tmp","tool_name":"Bash","tool_input":{"command":"echo"},"tool_response":{"stdout":"ok \ud83d\ude00"}}"#;
+ let output = run_hook_event(tmp.path(), "post-tool-use", payload);
+ assert!(output.status.success());
+ assert_eq!(spooled_body(tmp.path()).as_bytes(), payload);
+}
+
+#[test]
+fn malformed_payload_with_a_lone_surrogate_still_warns_without_spooling() {
+ // Invalid JSON stays invalid after the repair; a backslash before a
+ // multibyte character or at the end must not crash the hook.
+ for payload in [
+ r#"{"x":"\ud83d \é"#.as_bytes(),
+ r#"{"x":"\ud83d \"#.as_bytes(),
+ ] {
+ let tmp = tempfile::tempdir().expect("tempdir");
+ let output = run_hook(tmp.path(), payload);
+ assert!(output.status.success());
+ assert_eq!(output.stdout, b"{}\n");
+ assert_eq!(
+ String::from_utf8(output.stderr).expect("stderr utf8"),
+ "ai-memory hook warning: could not parse event payload as JSON; nothing was captured\n"
+ );
+ assert!(!tmp.path().join("hook-spool").exists());
+ }
+}