Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
fix(hooks): repair lone surrogates at shared ingress
AkitaOnRails30e7421380be
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 1df24c59..b06e0571 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -12,11 +12,11 @@ version in [`docs/changelog/`](docs/changelog/):## [Unreleased]### Fixed-- Kept native hook events whose JSON payload contains half of a UTF-16- surrogate pair as a `\u` escape, such as tool output cut in the middle of- an emoji. The hook could not parse the payload and dropped the whole event- with "nothing was captured"; the unpaired escape now becomes U+FFFD. The- shell and PowerShell hook scripts still post such a payload unchanged.+- Kept hook events whose JSON payload contains half of a UTF-16 surrogate pair+ as a `\u` escape, such as tool output cut in the middle of an emoji. Native,+ shell, PowerShell and direct hook clients now share the bounded server-side+ repair, replacing the unpaired escape with U+FFFD instead of dropping the+ whole event.- Added exact hover timestamps to `/web` page metadata, made the OKF upgradedialog keyboard accessible, and added a labelled main-navigation landmarkwith current-page state and narrow-screen wrapping.diff --git a/crates/ai-memory-cli/src/commands/hook.rs b/crates/ai-memory-cli/src/commands/hook.rsindex 2fe2458f..d8c97269 100644--- a/crates/ai-memory-cli/src/commands/hook.rs+++ b/crates/ai-memory-cli/src/commands/hook.rs@@ -1487,51 +1487,14 @@ fn parse_hook_payload(mut payload: String) -> serde_json::Result<(String, serde_// The repaired text becomes the spooled body: the server parses it// with `serde_json` too and would reject the original.Err(error) => {- let repaired = replace_lone_surrogate_escapes(&payload).ok_or(error)?;+ let repaired =+ ai_memory_hooks::replace_lone_surrogate_escapes(&payload).ok_or(error)?;let json = serde_json::from_str(&repaired)?;Ok((repaired, json))}}}-/// Replace each `\u` escape that names half of a UTF-16 surrogate pair without-/// its other half with U+FFFD. Harnesses that cut tool output mid-emoji send-/// such escapes, and `serde_json` rejects the whole payload for one of them.-/// An escaped backslash (`\\ud83d`) is text and stays as it is. Returns `None`-/// when nothing was replaced.-fn replace_lone_surrogate_escapes(text: &str) -> Option<String> {- let code_unit_at = |at: usize| {- let digits = text.as_bytes().get(at..at + 6)?.strip_prefix(b"\\u")?;- u16::from_str_radix(std::str::from_utf8(digits).ok()?, 16).ok()- };- let (mut out, mut copied, mut at) = (String::new(), 0, 0);- // `get` ends the scan instead of panicking when invalid JSON puts a- // backslash before a multibyte character or at the very end.- while let Some(offset) = text.get(at..).and_then(|rest| rest.find('\\')) {- at += offset;- match code_unit_at(at) {- // `\\`, `\n`, `\"` and the like: skip the escaped character too.- None => at += 2,- Some(0xD800..=0xDBFF) if matches!(code_unit_at(at + 6), Some(0xDC00..=0xDFFF)) => {- at += 12;- }- Some(0xD800..=0xDFFF) => {- out.push_str(&text[copied..at]);- out.push(char::REPLACEMENT_CHARACTER);- at += 6;- copied = at;- }- Some(_) => at += 6,- }- }- // A replacement always moves `copied` past its escape.- if copied == 0 {- return None;- }- out.push_str(&text[copied..]);- Some(out)-}-fn hook_context(agent: &str, raw: &serde_json::Value) -> (Option<String>, Option<String>) {let (_, session_id) = canonical_context(raw);let cwd = resolve_hook_cwd_with(agent, raw, env_lookup, || std::env::current_dir().ok());diff --git a/crates/ai-memory-hooks/src/json_repair.rs b/crates/ai-memory-hooks/src/json_repair.rsnew file mode 100644index 00000000..cfe627d3--- /dev/null+++ b/crates/ai-memory-hooks/src/json_repair.rs@@ -0,0 +1,53 @@+//! Narrow repair for hook JSON cut between UTF-16 surrogate escapes.++/// Replace each `\u` escape that names half of a UTF-16 surrogate pair without+/// its other half with U+FFFD. Harnesses that cut tool output mid-emoji can+/// emit such JSON, and `serde_json` rejects the whole event for one bad escape.+/// An escaped backslash (`\\ud83d`) is text and stays unchanged.+///+/// Returns `None` when nothing was replaced.+#[must_use]+pub fn replace_lone_surrogate_escapes(text: &str) -> Option<String> {+ let code_unit_at = |at: usize| {+ let digits = text.as_bytes().get(at..at + 6)?.strip_prefix(b"\\u")?;+ u16::from_str_radix(std::str::from_utf8(digits).ok()?, 16).ok()+ };+ let (mut out, mut copied, mut at) = (String::new(), 0, 0);+ while let Some(offset) = text.get(at..).and_then(|rest| rest.find('\\')) {+ at += offset;+ match code_unit_at(at) {+ // `\\`, `\n`, `\"` and the like: skip the escaped character too.+ None => at += 2,+ Some(0xD800..=0xDBFF) if matches!(code_unit_at(at + 6), Some(0xDC00..=0xDFFF)) => {+ at += 12;+ }+ Some(0xD800..=0xDFFF) => {+ out.push_str(&text[copied..at]);+ out.push(char::REPLACEMENT_CHARACTER);+ at += 6;+ copied = at;+ }+ Some(_) => at += 6,+ }+ }+ if copied == 0 {+ return None;+ }+ out.push_str(&text[copied..]);+ Some(out)+}++#[cfg(test)]+mod tests {+ use super::replace_lone_surrogate_escapes;++ #[test]+ fn repairs_only_unpaired_surrogate_escapes() {+ let raw = r#"{"text":"\ud83d \ud83d\ude00 \\ud83d \\\ude00"}"#;+ assert_eq!(+ replace_lone_surrogate_escapes(raw).as_deref(),+ Some(r#"{"text":"� \ud83d\ude00 \\ud83d \\�"}"#)+ );+ assert!(replace_lone_surrogate_escapes(r#"{"text":"\ud83d\ude00"}"#).is_none());+ }+}diff --git a/crates/ai-memory-hooks/src/lib.rs b/crates/ai-memory-hooks/src/lib.rsindex 6d6dffd6..6be97b1c 100644--- a/crates/ai-memory-hooks/src/lib.rs+++ b/crates/ai-memory-hooks/src/lib.rs@@ -27,6 +27,7 @@ mod brief_tiers;pub mod capture_policy;pub mod drop_list;mod grants;+mod json_repair;pub mod log;pub mod payload;mod recap;@@ -54,6 +55,7 @@ pub use capture_policy::{glob_segment_matches, glob_walk, repository_admits_capture, tool_call_payload, valid_call_id,};pub use drop_list::IngestDropList;+pub use json_repair::replace_lone_surrogate_escapes;pub use payload::{HookEnvelope, HookEvent, NOTIFICATION_EXCERPT_MAX_BYTES, POST_COMPACTION_EXCERPT_MAX_BYTES,USER_PROMPT_EXCERPT_MAX_BYTES, agent_from_payload, cap_lifecycle_body_for_client,diff --git a/crates/ai-memory-hooks/src/router.rs b/crates/ai-memory-hooks/src/router.rsindex 22a02fd1..09504d22 100644--- a/crates/ai-memory-hooks/src/router.rs+++ b/crates/ai-memory-hooks/src/router.rs@@ -24,12 +24,14 @@ use ai_memory_store::{lookup_existing_workspace,};use ai_memory_wiki::{AdmissionContext, AdmissionOp, Wiki};-use axum::Json;-use axum::Router;+use axum::body::{Body, to_bytes};use axum::extract::{Query, State};-use axum::http::{HeaderMap, StatusCode};+use axum::http::{HeaderMap, Request, StatusCode};+use axum::middleware::{Next, from_fn};use axum::response::IntoResponse;+use axum::response::Response;use axum::routing::{get, post};+use axum::{Json, Router};use jiff::Timestamp;use serde::{Deserialize, Serialize};use tracing::{debug, info, warn};@@ -65,6 +67,9 @@ pub const DEFAULT_INGEST_GATE_MAX_ENTRIES: usize = 4096;/// allocate/process an unbounded vector of hook events.pub const MAX_HOOK_BATCH_ITEMS: usize = 256;+/// Keep repair buffering at the server's global machine-route request limit.+const MAX_HOOK_BODY_BYTES: usize = 10 * 1024 * 1024;+/// Upper bound for reject-policy admission on the synchronous `/handoff`/// path. The shortest shipped client deadline is the shell hook's one-second/// curl timeout, so the server must decide earlier or leave the baton open;@@ -631,13 +636,42 @@ fn admission_skips(/// (the read-only compaction recap for a "reload context" prompt).pub fn hook_router(state: HookState) -> Router {Router::new()- .route("/hook", post(handle_hook))+ .route(+ "/hook",+ post(handle_hook).route_layer(from_fn(repair_hook_json)),+ ).route("/hook/batch", post(handle_hook_batch)).route("/handoff", get(handle_handoff)).route("/recap", get(crate::recap::handle_recap)).with_state(Arc::new(state))}+/// Repair the one invalid-JSON shape first-party harnesses can produce when+/// they truncate an escaped emoji between its UTF-16 surrogate halves. The+/// normal `Json` extractor still owns media-type validation and all parsing;+/// valid bodies pass through byte-for-byte.+async fn repair_hook_json(request: Request<Body>, next: Next) -> Response {+ let (parts, body) = request.into_parts();+ let bytes = match to_bytes(body, MAX_HOOK_BODY_BYTES).await {+ Ok(bytes) => bytes,+ Err(_) => return StatusCode::PAYLOAD_TOO_LARGE.into_response(),+ };+ let bytes = repair_hook_json_bytes(bytes);+ next.run(Request::from_parts(parts, Body::from(bytes)))+ .await+}++fn repair_hook_json_bytes(bytes: axum::body::Bytes) -> axum::body::Bytes {+ if serde_json::from_slice::<serde_json::Value>(&bytes).is_err() {+ std::str::from_utf8(&bytes)+ .ok()+ .and_then(crate::replace_lone_surrogate_escapes)+ .map_or(bytes, axum::body::Bytes::from)+ } else {+ bytes+ }+}+/// Unix milliseconds for an ingest metric stamp, saturating rather than/// failing: a clock before the epoch or past `u64` must not cost an event.fn now_unix_ms() -> u64 {@@ -6451,6 +6485,16 @@ pub(crate) mod tests {}}+ #[test]+ fn hook_ingress_repairs_lone_surrogate_before_json_extraction() {+ let raw = axum::body::Bytes::from_static(+ br#"{"session_id":"lone","tool_response":{"stdout":"ok \ud83d tail"}}"#,+ );+ let repaired = repair_hook_json_bytes(raw);+ let value: serde_json::Value = serde_json::from_slice(&repaired).unwrap();+ assert_eq!(value["tool_response"]["stdout"], "ok \u{FFFD} tail");+ }+/// A consolidator whose provider panics if the hook path ever calls it.fn with_hook_consolidator(state: &mut HookState) {state.consolidator = Some(Arc::new(Consolidator::new(