Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

fix(hooks): repair lone surrogates at shared ingress

AkitaOnRails30e7421380be
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1df24c59..b06e0571 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,11 +12,11 @@ version in [`docs/changelog/`](docs/changelog/):
## [Unreleased]
### Fixed
-- Kept native hook events whose JSON payload contains half of a UTF-16
- surrogate pair as a `\u` escape, such as tool output cut in the middle of
- an emoji. The hook could not parse the payload and dropped the whole event
- with "nothing was captured"; the unpaired escape now becomes U+FFFD. The
- shell and PowerShell hook scripts still post such a payload unchanged.
+- Kept hook events whose JSON payload contains half of a UTF-16 surrogate pair
+ as a `\u` escape, such as tool output cut in the middle of an emoji. Native,
+ shell, PowerShell and direct hook clients now share the bounded server-side
+ repair, replacing the unpaired escape with U+FFFD instead of dropping the
+ whole event.
- Added exact hover timestamps to `/web` page metadata, made the OKF upgrade
dialog keyboard accessible, and added a labelled main-navigation landmark
with current-page state and narrow-screen wrapping.
diff --git a/crates/ai-memory-cli/src/commands/hook.rs b/crates/ai-memory-cli/src/commands/hook.rs
index 2fe2458f..d8c97269 100644
--- a/crates/ai-memory-cli/src/commands/hook.rs
+++ b/crates/ai-memory-cli/src/commands/hook.rs
@@ -1487,51 +1487,14 @@ fn parse_hook_payload(mut payload: String) -> serde_json::Result<(String, serde_
// The repaired text becomes the spooled body: the server parses it
// with `serde_json` too and would reject the original.
Err(error) => {
- let repaired = replace_lone_surrogate_escapes(&payload).ok_or(error)?;
+ let repaired =
+ ai_memory_hooks::replace_lone_surrogate_escapes(&payload).ok_or(error)?;
let json = serde_json::from_str(&repaired)?;
Ok((repaired, json))
}
}
}
-/// Replace each `\u` escape that names half of a UTF-16 surrogate pair without
-/// its other half with U+FFFD. Harnesses that cut tool output mid-emoji send
-/// such escapes, and `serde_json` rejects the whole payload for one of them.
-/// An escaped backslash (`\\ud83d`) is text and stays as it is. Returns `None`
-/// when nothing was replaced.
-fn replace_lone_surrogate_escapes(text: &str) -> Option<String> {
- let code_unit_at = |at: usize| {
- let digits = text.as_bytes().get(at..at + 6)?.strip_prefix(b"\\u")?;
- u16::from_str_radix(std::str::from_utf8(digits).ok()?, 16).ok()
- };
- let (mut out, mut copied, mut at) = (String::new(), 0, 0);
- // `get` ends the scan instead of panicking when invalid JSON puts a
- // backslash before a multibyte character or at the very end.
- while let Some(offset) = text.get(at..).and_then(|rest| rest.find('\\')) {
- at += offset;
- match code_unit_at(at) {
- // `\\`, `\n`, `\"` and the like: skip the escaped character too.
- None => at += 2,
- Some(0xD800..=0xDBFF) if matches!(code_unit_at(at + 6), Some(0xDC00..=0xDFFF)) => {
- at += 12;
- }
- Some(0xD800..=0xDFFF) => {
- out.push_str(&text[copied..at]);
- out.push(char::REPLACEMENT_CHARACTER);
- at += 6;
- copied = at;
- }
- Some(_) => at += 6,
- }
- }
- // A replacement always moves `copied` past its escape.
- if copied == 0 {
- return None;
- }
- out.push_str(&text[copied..]);
- Some(out)
-}
-
fn hook_context(agent: &str, raw: &serde_json::Value) -> (Option<String>, Option<String>) {
let (_, session_id) = canonical_context(raw);
let cwd = resolve_hook_cwd_with(agent, raw, env_lookup, || std::env::current_dir().ok());
diff --git a/crates/ai-memory-hooks/src/json_repair.rs b/crates/ai-memory-hooks/src/json_repair.rs
new file mode 100644
index 00000000..cfe627d3
--- /dev/null
+++ b/crates/ai-memory-hooks/src/json_repair.rs
@@ -0,0 +1,53 @@
+//! Narrow repair for hook JSON cut between UTF-16 surrogate escapes.
+
+/// Replace each `\u` escape that names half of a UTF-16 surrogate pair without
+/// its other half with U+FFFD. Harnesses that cut tool output mid-emoji can
+/// emit such JSON, and `serde_json` rejects the whole event for one bad escape.
+/// An escaped backslash (`\\ud83d`) is text and stays unchanged.
+///
+/// Returns `None` when nothing was replaced.
+#[must_use]
+pub fn replace_lone_surrogate_escapes(text: &str) -> Option<String> {
+ let code_unit_at = |at: usize| {
+ let digits = text.as_bytes().get(at..at + 6)?.strip_prefix(b"\\u")?;
+ u16::from_str_radix(std::str::from_utf8(digits).ok()?, 16).ok()
+ };
+ let (mut out, mut copied, mut at) = (String::new(), 0, 0);
+ while let Some(offset) = text.get(at..).and_then(|rest| rest.find('\\')) {
+ at += offset;
+ match code_unit_at(at) {
+ // `\\`, `\n`, `\"` and the like: skip the escaped character too.
+ None => at += 2,
+ Some(0xD800..=0xDBFF) if matches!(code_unit_at(at + 6), Some(0xDC00..=0xDFFF)) => {
+ at += 12;
+ }
+ Some(0xD800..=0xDFFF) => {
+ out.push_str(&text[copied..at]);
+ out.push(char::REPLACEMENT_CHARACTER);
+ at += 6;
+ copied = at;
+ }
+ Some(_) => at += 6,
+ }
+ }
+ if copied == 0 {
+ return None;
+ }
+ out.push_str(&text[copied..]);
+ Some(out)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::replace_lone_surrogate_escapes;
+
+ #[test]
+ fn repairs_only_unpaired_surrogate_escapes() {
+ let raw = r#"{"text":"\ud83d \ud83d\ude00 \\ud83d \\\ude00"}"#;
+ assert_eq!(
+ replace_lone_surrogate_escapes(raw).as_deref(),
+ Some(r#"{"text":"� \ud83d\ude00 \\ud83d \\�"}"#)
+ );
+ assert!(replace_lone_surrogate_escapes(r#"{"text":"\ud83d\ude00"}"#).is_none());
+ }
+}
diff --git a/crates/ai-memory-hooks/src/lib.rs b/crates/ai-memory-hooks/src/lib.rs
index 6d6dffd6..6be97b1c 100644
--- a/crates/ai-memory-hooks/src/lib.rs
+++ b/crates/ai-memory-hooks/src/lib.rs
@@ -27,6 +27,7 @@ mod brief_tiers;
pub mod capture_policy;
pub mod drop_list;
mod grants;
+mod json_repair;
pub mod log;
pub mod payload;
mod recap;
@@ -54,6 +55,7 @@ pub use capture_policy::{
glob_segment_matches, glob_walk, repository_admits_capture, tool_call_payload, valid_call_id,
};
pub use drop_list::IngestDropList;
+pub use json_repair::replace_lone_surrogate_escapes;
pub use payload::{
HookEnvelope, HookEvent, NOTIFICATION_EXCERPT_MAX_BYTES, POST_COMPACTION_EXCERPT_MAX_BYTES,
USER_PROMPT_EXCERPT_MAX_BYTES, agent_from_payload, cap_lifecycle_body_for_client,
diff --git a/crates/ai-memory-hooks/src/router.rs b/crates/ai-memory-hooks/src/router.rs
index 22a02fd1..09504d22 100644
--- a/crates/ai-memory-hooks/src/router.rs
+++ b/crates/ai-memory-hooks/src/router.rs
@@ -24,12 +24,14 @@ use ai_memory_store::{
lookup_existing_workspace,
};
use ai_memory_wiki::{AdmissionContext, AdmissionOp, Wiki};
-use axum::Json;
-use axum::Router;
+use axum::body::{Body, to_bytes};
use axum::extract::{Query, State};
-use axum::http::{HeaderMap, StatusCode};
+use axum::http::{HeaderMap, Request, StatusCode};
+use axum::middleware::{Next, from_fn};
use axum::response::IntoResponse;
+use axum::response::Response;
use axum::routing::{get, post};
+use axum::{Json, Router};
use jiff::Timestamp;
use serde::{Deserialize, Serialize};
use tracing::{debug, info, warn};
@@ -65,6 +67,9 @@ pub const DEFAULT_INGEST_GATE_MAX_ENTRIES: usize = 4096;
/// allocate/process an unbounded vector of hook events.
pub const MAX_HOOK_BATCH_ITEMS: usize = 256;
+/// Keep repair buffering at the server's global machine-route request limit.
+const MAX_HOOK_BODY_BYTES: usize = 10 * 1024 * 1024;
+
/// Upper bound for reject-policy admission on the synchronous `/handoff`
/// path. The shortest shipped client deadline is the shell hook's one-second
/// curl timeout, so the server must decide earlier or leave the baton open;
@@ -631,13 +636,42 @@ fn admission_skips(
/// (the read-only compaction recap for a "reload context" prompt).
pub fn hook_router(state: HookState) -> Router {
Router::new()
- .route("/hook", post(handle_hook))
+ .route(
+ "/hook",
+ post(handle_hook).route_layer(from_fn(repair_hook_json)),
+ )
.route("/hook/batch", post(handle_hook_batch))
.route("/handoff", get(handle_handoff))
.route("/recap", get(crate::recap::handle_recap))
.with_state(Arc::new(state))
}
+/// Repair the one invalid-JSON shape first-party harnesses can produce when
+/// they truncate an escaped emoji between its UTF-16 surrogate halves. The
+/// normal `Json` extractor still owns media-type validation and all parsing;
+/// valid bodies pass through byte-for-byte.
+async fn repair_hook_json(request: Request<Body>, next: Next) -> Response {
+ let (parts, body) = request.into_parts();
+ let bytes = match to_bytes(body, MAX_HOOK_BODY_BYTES).await {
+ Ok(bytes) => bytes,
+ Err(_) => return StatusCode::PAYLOAD_TOO_LARGE.into_response(),
+ };
+ let bytes = repair_hook_json_bytes(bytes);
+ next.run(Request::from_parts(parts, Body::from(bytes)))
+ .await
+}
+
+fn repair_hook_json_bytes(bytes: axum::body::Bytes) -> axum::body::Bytes {
+ if serde_json::from_slice::<serde_json::Value>(&bytes).is_err() {
+ std::str::from_utf8(&bytes)
+ .ok()
+ .and_then(crate::replace_lone_surrogate_escapes)
+ .map_or(bytes, axum::body::Bytes::from)
+ } else {
+ bytes
+ }
+}
+
/// Unix milliseconds for an ingest metric stamp, saturating rather than
/// failing: a clock before the epoch or past `u64` must not cost an event.
fn now_unix_ms() -> u64 {
@@ -6451,6 +6485,16 @@ pub(crate) mod tests {
}
}
+ #[test]
+ fn hook_ingress_repairs_lone_surrogate_before_json_extraction() {
+ let raw = axum::body::Bytes::from_static(
+ br#"{"session_id":"lone","tool_response":{"stdout":"ok \ud83d tail"}}"#,
+ );
+ let repaired = repair_hook_json_bytes(raw);
+ let value: serde_json::Value = serde_json::from_slice(&repaired).unwrap();
+ assert_eq!(value["tool_response"]["stdout"], "ok \u{FFFD} tail");
+ }
+
/// A consolidator whose provider panics if the hook path ever calls it.
fn with_hook_consolidator(state: &mut HookState) {
state.consolidator = Some(Arc::new(Consolidator::new(