Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

fix(llm): preserve literal reasoning tags in structured replies

Samir Hanna Verza7052fe73fb99
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e65cba04..db30dd4d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,6 +13,10 @@ version in [`docs/changelog/`](docs/changelog/):
### Fixed
+- Preserved literal reasoning tags and their contents inside structured JSON
+ strings when parsing OpenAI-compatible tolerant replies and Anthropic text
+ replies. Reasoning blocks outside the JSON are still removed, including
+ when a strict OpenAI-compatible request falls back to tolerant parsing (#PR).
- Refused empty and whitespace-only `--workspace` and `--project` flags at the
shared CLI scope-resolution choke point (`resolve_scope`,
`resolve_scope_noting_marker_project`, `resolve_scope_with_marker`, and
diff --git a/crates/ai-memory-llm/src/openai_compat.rs b/crates/ai-memory-llm/src/openai_compat.rs
index cf5791ba..5c32ec41 100644
--- a/crates/ai-memory-llm/src/openai_compat.rs
+++ b/crates/ai-memory-llm/src/openai_compat.rs
@@ -22,23 +22,13 @@ use crate::provider::LlmProvider;
use crate::text::{suffix_within_bytes, truncate_with_ellipsis};
use crate::types::{ChatRequest, ChatResponse, LlmOperationId};
-// Compiled once. Matches <think>, <thinking>, <analysis>, <reasoning> blocks
-// (case-insensitive, non-greedy, DOTALL) that reasoning models emit before
-// the JSON payload. These blocks often contain `{` braces that confuse the
-// balanced-brace extractor when unstripped.
-//
-// Each tag is listed explicitly because the `regex` crate does not support
-// backreferences — we cannot write `<(tag)>.*?</\1>`.
-static REASONING_BLOCK_RE: LazyLock<Regex> = LazyLock::new(|| {
- Regex::new(
- r"(?is)(?:<think>.*?</think>|<thinking>.*?</thinking>|<analysis>.*?</analysis>|<reasoning>.*?</reasoning>)",
- )
- .unwrap()
+// Scan reasoning tags and JSON starts in order.
+// A reasoning block consumes its draft JSON, but a JSON value encountered
+// first must survive intact even when its strings contain reasoning tags.
+static REASONING_OR_JSON_RE: LazyLock<Regex> = LazyLock::new(|| {
+ Regex::new(r"(?i)(?:</?(?:think|thinking|analysis|reasoning)>|[\{\[])").unwrap()
});
-// The first closing reasoning tag. Chat templates that open the reasoning
-// block themselves (DeepSeek-R1, QwQ, Qwen3 thinking models) return the
-// reasoning followed by a bare `</think>`, with no opening tag to pair it.
static REASONING_CLOSE_RE: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"(?i)</(?:think|thinking|analysis|reasoning)>").unwrap());
@@ -47,25 +37,61 @@ static CODE_FENCE_RE: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"(?s)\A\s*```(?:json)?\s*\n(.*?)\n\s*```\s*\z").unwrap());
/// Remove `<think>`, `<thinking>`, `<analysis>`, and `<reasoning>` blocks
-/// (case-insensitive) from `s`, then unwrap any surrounding markdown code
-/// fence. Returns a freshly allocated `String` so the caller can choose
-/// between the cleaned and the raw form.
+/// (case-insensitive) outside JSON values in `s`, then unwrap any surrounding
+/// markdown code fence. JSON string contents are copied verbatim.
pub(crate) fn strip_reasoning_blocks(s: &str) -> String {
- let stripped = REASONING_BLOCK_RE.replace_all(s, "");
- // A closing tag left after the paired blocks are gone ends a reasoning
- // section whose opening tag the chat template supplied. Drop that
- // section too, unless the text before it already looks like the JSON
- // answer (a literal closing tag inside a JSON string must survive).
- let stripped = match REASONING_CLOSE_RE.find(stripped.as_ref()) {
- Some(close)
- if !stripped[..close.start()]
- .trim_start()
- .starts_with(['{', '[', '`']) =>
- {
- std::borrow::Cow::Owned(stripped[close.end()..].to_string())
+ let mut stripped = String::with_capacity(s.len());
+ let mut cursor = 0;
+ let mut saw_unopened_close = false;
+ let mut no_closing_tag: Vec<&str> = Vec::new();
+ while let Some(token) = REASONING_OR_JSON_RE.find_at(s, cursor) {
+ stripped.push_str(&s[cursor..token.start()]);
+ cursor = token.end();
+ if matches!(token.as_str(), "{" | "[") {
+ // JSON strings may contain these literal tags (RFC 8259 §7):
+ // https://www.rfc-editor.org/rfc/rfc8259#section-7
+ // Let serde locate the end, honoring escapes and nested values,
+ // rather than matching a tag inside the answer as a wrapper.
+ let mut json = serde_json::Deserializer::from_str(&s[token.start()..])
+ .into_iter::<serde_json::Value>();
+ if matches!(json.next(), Some(Ok(_))) {
+ cursor = token.start() + json.byte_offset();
+ stripped.push_str(&s[token.start()..cursor]);
+ } else {
+ stripped.push_str(token.as_str());
+ }
+ } else if token.as_str().starts_with("</") {
+ // Some chat templates supply the opening tag themselves. Keep
+ // the existing first-close rule, unless the prefix already looks
+ // like the answer. Closing tags inside JSON never reach this arm.
+ if !saw_unopened_close && !stripped.trim_start().starts_with(['{', '[', '`']) {
+ stripped.clear();
+ } else {
+ stripped.push_str(token.as_str());
+ }
+ saw_unopened_close = true;
+ } else {
+ let tag = &token.as_str()[1..token.len() - 1];
+ if !no_closing_tag
+ .iter()
+ .any(|seen| seen.eq_ignore_ascii_case(tag))
+ {
+ let close = REASONING_CLOSE_RE
+ .find_iter(&s[cursor..])
+ .find(|close| close.as_str()[2..close.len() - 1].eq_ignore_ascii_case(tag));
+ if let Some(close) = close {
+ cursor += close.end();
+ continue;
+ }
+ // The cursor only moves forward. Cache each of the four
+ // missing closes so repeated unfinished tags cannot rescan
+ // the remaining response once per opening tag.
+ no_closing_tag.push(tag);
+ }
+ stripped.push_str(token.as_str());
}
- _ => stripped,
- };
+ }
+ stripped.push_str(&s[cursor..]);
// If the entire remaining payload is wrapped in a code fence, unwrap it
// so `serde_json::from_str` can see the bare JSON directly.
if let Some(caps) = CODE_FENCE_RE.captures(stripped.as_ref()) {
@@ -756,6 +782,40 @@ mod tests {
);
}
+ #[test]
+ fn reasoning_tags_in_json_members_and_nested_values_are_kept() {
+ let input = r#"{"<think>key</think>":[{"body":"<analysis>nested</analysis>"}]}"#;
+ assert_eq!(strip_reasoning_blocks(input), input);
+ }
+
+ /// An unmatched draft brace in reasoning must not hide the template's
+ /// closing tag or prevent the following answer from being protected.
+ #[test]
+ fn malformed_reasoning_draft_still_reaches_the_answer() {
+ let answer = r#"{"body":"<think>literal</think>"}"#;
+ for prefix in [
+ "Rough draft {not JSON</think>\n",
+ "<analysis>Rough draft {not JSON</analysis>\n",
+ ] {
+ assert_eq!(strip_reasoning_blocks(&format!("{prefix}{answer}")), answer);
+ }
+ }
+
+ #[test]
+ fn a_literal_opening_tag_does_not_consume_a_later_reasoning_block() {
+ let answer = r#"{"body":"an opening <think> tag"}"#;
+ assert_eq!(
+ strip_reasoning_blocks(&format!("{answer}<think>actual reasoning</think>")),
+ answer
+ );
+ }
+
+ #[test]
+ fn many_unclosed_reasoning_tags_preserve_prose() {
+ let input = "<think>{<thinking>[<analysis>{<REASONING>[".repeat(2048);
+ assert_eq!(strip_reasoning_blocks(&input), input);
+ }
+
/// `<analysis>` prefix is also stripped.
#[test]
fn strip_analysis_block() {
diff --git a/crates/ai-memory-llm/tests/suite/anthropic_provider.rs b/crates/ai-memory-llm/tests/suite/anthropic_provider.rs
index bb764c63..37d9b67a 100644
--- a/crates/ai-memory-llm/tests/suite/anthropic_provider.rs
+++ b/crates/ai-memory-llm/tests/suite/anthropic_provider.rs
@@ -103,6 +103,24 @@ async fn an_unforced_call_reads_json_after_thinking_or_in_prose() {
assert_eq!(outcome.unwrap(), json!({"answer": "ok"}));
}
+#[rstest::rstest]
+#[case::plain("", "")]
+#[case::fenced("```json\n", "\n```")]
+#[case::prose("Here it is: ", "")]
+#[case::reasoning("<think>Draft {\"answer\":\"discard\"}</think>\n", "")]
+#[tokio::test]
+async fn unforced_text_reply_preserves_literal_reasoning_tags(
+ #[case] prefix: &str,
+ #[case] suffix: &str,
+) {
+ let expected =
+ json!({"answer": "Keep <analysis>literal text</analysis> and <think>quoted text</think>."});
+ let content = format!("{prefix}{expected}{suffix}");
+ let (outcome, body) = structured_call("claude-opus-5-5", text_reply(&content)).await;
+ assert!(body.get("tool_choice").is_none());
+ assert_eq!(outcome.unwrap(), expected);
+}
+
#[tokio::test]
async fn a_reply_without_a_tool_call_or_json_is_still_an_error() {
let prose = text_reply("I cannot produce that.");
diff --git a/crates/ai-memory-llm/tests/suite/openai_compat_strict.rs b/crates/ai-memory-llm/tests/suite/openai_compat_strict.rs
index 253ad52a..81f1ec14 100644
--- a/crates/ai-memory-llm/tests/suite/openai_compat_strict.rs
+++ b/crates/ai-memory-llm/tests/suite/openai_compat_strict.rs
@@ -57,6 +57,49 @@ fn body_with_content(content: &str) -> serde_json::Value {
})
}
+/// Reasoning tags inside the answer are data, including when the endpoint
+/// falls back from strict output or wraps the answer in prose/reasoning.
+#[rstest::rstest]
+#[case::plain("", "", false, 1)]
+#[case::fenced("```json\n", "\n```", false, 1)]
+#[case::prose("Here is the result: ", "", false, 1)]
+#[case::reasoning("<think>Draft {\"answer\":\"discard\"}</think>\n", "", false, 1)]
+#[case::unopened("Draft {\"answer\":\"discard\"}</think>\n", "", false, 1)]
+#[case::strict_fallback("```json\n", "\n```", true, 2)]
+#[case::strict_control("", "", true, 1)]
+#[tokio::test]
+async fn structured_reply_preserves_literal_reasoning_tags(
+ #[case] prefix: &str,
+ #[case] suffix: &str,
+ #[case] strict: bool,
+ #[case] requests: u64,
+) {
+ let expected = json!({"answer": concat!(
+ "Keep <think>thought</think>, <thinking>quoted</thinking>, ",
+ "<analysis>analysis</analysis>, <REASONING>reason</REASONING>, ",
+ "a lone </think>, \"quotes\", \\paths, {braces}, and café."
+ )});
+ let content = format!("{prefix}{expected}{suffix}");
+ let server = MockServer::start().await;
+ Mock::given(method("POST"))
+ .and(path("/v1/chat/completions"))
+ .respond_with(ResponseTemplate::new(200).set_body_json(body_with_content(&content)))
+ .expect(requests)
+ .mount(&server)
+ .await;
+ let provider = OpenAiCompatProvider::new(server.uri(), None, "model-x")
+ .unwrap()
+ .with_strict(strict);
+ let actual = provider
+ .complete_structured_raw(
+ tiny_request(),
+ json!({"type":"object", "properties":{"answer":{"type":"string"}}}),
+ )
+ .await
+ .expect("a complete JSON answer");
+ assert_eq!(actual, expected);
+}
+
/// Counts every request hitting the mock server so a test can assert
/// "only one call was made" (strict succeeded) vs "two calls were made"
/// (strict raw, then tolerant fallback). The response itself is cloned
diff --git a/docs/llm-providers.md b/docs/llm-providers.md
index 2f5f0e91..e7979c6c 100644
--- a/docs/llm-providers.md
+++ b/docs/llm-providers.md
@@ -381,6 +381,12 @@ also set `COPILOT_GITHUB_TOKEN`, `GH_TOKEN`, or `GITHUB_TOKEN` on the server.
> endpoint explicitly rejects that field or returns a malformed shape. Set
> `AI_MEMORY_LLM_COMPAT_STRICT=false` only for an incompatible endpoint.
+The tolerant parser removes `<think>`, `<thinking>`, `<analysis>` and
+`<reasoning>` wrappers outside the JSON answer and unwraps Markdown fences.
+Literal tags inside JSON strings remain intact, so summaries can quote prompt
+templates or other tagged text. Anthropic's structured text replies use the
+same parsing rules when the provider uses automatic tool choice.
+
> [!TIP]
> **Thinking-capable local engines (vLLM / SGLang serving Qwen3-class
> models).** Such engines spend the output budget on a reasoning pass before