Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix: A9 — tag protector discards wrap on placeholder loss
chopratejas6aacd4805add
diff --git a/crates/headroom-core/proptest-regressions/transforms/tag_protector.txt b/crates/headroom-core/proptest-regressions/transforms/tag_protector.txtnew file mode 100644index 000000000..760155ae8--- /dev/null+++ b/crates/headroom-core/proptest-regressions/transforms/tag_protector.txt@@ -0,0 +1,9 @@+# Seeds for failure cases proptest has generated in the past. It is+# automatically read and these particular cases re-run before any+# novel cases are generated.+#+# It is recommended to check this file in to source control so that+# everyone who runs the test benefits from these saved cases.+cc 8fbea33614d948018222b12893746cadfc057a807a80f3e8b63af540614d1eb0 # shrinks to content = "<a>"+cc e499bfe23a63b8a21d0d8105141c83e83a7f3a1484c19bcf5be0732c7c1bd9c3 # shrinks to content = "</"+cc 70ba971eb9c85e814763d0af359de0a4a967e8947caf837d4efe751a37292659 # shrinks to content = "</", compressed = ""diff --git a/crates/headroom-core/src/transforms/tag_protector.rs b/crates/headroom-core/src/transforms/tag_protector.rsindex 2b967b961..2728022b7 100644--- a/crates/headroom-core/src/transforms/tag_protector.rs+++ b/crates/headroom-core/src/transforms/tag_protector.rs@@ -324,6 +324,13 @@ fn parse_tag_at(bytes: &[u8], start: usize) -> TagParse {if is_close {i += 1;}+ // After consuming a possible '/' we may be at end-of-input+ // (e.g. literal `</` with nothing after). Guard the bounds+ // before indexing into `bytes[i]` for the name-start check —+ // proptest discovered the OOB on input `</`.+ if i >= n {+ return TagParse::NotTag;+ }let name_start = i;if !is_name_start(bytes[i]) {return TagParse::NotTag;@@ -638,45 +645,94 @@ fn emit_output(/// Restore protected tag spans after the compressor ran on the/// cleaned text.///+/// # Hotfix-A9 — discard-wrap semantics+////// If a placeholder went missing during compression (the compressor-/// stripped or rewrote it) the corresponding original is appended to-/// the output rather than dropped — same fallback semantics as the-/// Python original. A `tracing::warn!` is emitted at compile-time--/// optional verbosity so prod can scrape lossy-compression incidents.+/// stripped or rewrote it) the wrap is **discarded**: the compressed+/// text flows downstream as-is and the original tag bytes are NOT+/// re-injected anywhere. This is a deliberate behavior change vs the+/// original "append the orphan tag at the trailing edge" fallback,+/// which produced silently malformed XML (an opening tag with no+/// closing tag and no body) on ~350 production requests over 9 days.+///+/// An ERROR-level `tracing` event with structured fields+/// (`event = tag_protector_placeholder_lost`, `tag_preview`,+/// `compressed_length`, optional `request_id`) is emitted per lost+/// placeholder so operators can alert on the corruption rather than+/// having it disappear into a WARN line. Token validation downstream+/// is responsible for catching cases where the discard regressed the+/// final output vs the original input.+///+/// Invariants enforced:+/// 1. Symmetry — never emit asymmetric tag counts (the input either+/// survives both opens and closes via successful substitution, or+/// neither survives via discard).+/// 2. No orphan tag injection — `restore_tags` adds bytes only as part+/// of placeholder substitution. No appends, no prepends, no+/// whitespace insertion outside placeholder substitutions.+/// 3. Idempotence on missing placeholders — if every placeholder is+/// absent from `compressed`, the function returns `compressed`+/// byte-for-byte unchanged.pub fn restore_tags(text: &str, blocks: &[(String, String)]) -> String {+ restore_tags_with_request_id(text, blocks, None)+}++/// Variant of [`restore_tags`] that threads an optional `request_id`+/// into the structured ERROR log emitted on placeholder loss. The+/// PyO3 binding currently calls [`restore_tags`] (no request id);+/// this entry point exists so the proxy layer can wire request+/// context through once it has one available end-to-end.+pub fn restore_tags_with_request_id(+ text: &str,+ blocks: &[(String, String)],+ request_id: Option<&str>,+) -> String {if blocks.is_empty() {return text.to_string();}let mut result = text.to_string();- let mut tail_appends: Vec<&str> = Vec::new();+ let mut lost_count: usize = 0;+ let compressed_length = text.len();for (placeholder, original) in blocks {if result.contains(placeholder.as_str()) {result = result.replace(placeholder.as_str(), original);} else {- tag_lost_warn(original);- tail_appends.push(original.as_str());- }- }-- if !tail_appends.is_empty() {- for original in tail_appends {- result.push('\n');- result.push_str(original);+ lost_count += 1;+ tag_lost_error(original, compressed_length, request_id);}}-+ // No tail_appends. The compressed text is returned with the+ // wraps for the lost placeholders fully discarded — never+ // appended back as orphan opens (Hotfix-A9).+ let _ = lost_count; // surfaced via the per-event log; aggregate+ // counters live on the stats sidecar in+ // callers that have one.result}#[inline(never)]-fn tag_lost_warn(original: &str) {+fn tag_lost_error(original: &str, compressed_length: usize, request_id: Option<&str>) {let preview: String = original.chars().take(80).collect();- tracing::warn!(- target: "headroom::tag_protector",- preview = %preview,- "tag placeholder lost during compression, appending original"- );+ match request_id {+ Some(rid) => tracing::error!(+ target: "headroom::tag_protector",+ event = "tag_protector_placeholder_lost",+ tag_preview = %preview,+ compressed_length = compressed_length,+ request_id = %rid,+ action = "discarded_wrap",+ "tag placeholder lost during compression — wrap discarded"+ ),+ None => tracing::error!(+ target: "headroom::tag_protector",+ event = "tag_protector_placeholder_lost",+ tag_preview = %preview,+ compressed_length = compressed_length,+ action = "discarded_wrap",+ "tag placeholder lost during compression — wrap discarded"+ ),+ }}// ─── Tiny byte-search helper ──────────────────────────────────────────@@ -824,13 +880,57 @@ mod tests {}#[test]- fn restore_lost_placeholder_appended() {+ fn restore_lost_placeholder_discards_wrap() {+ // Hotfix-A9: when a placeholder is missing from the compressed+ // text, the wrap is DISCARDED — the compressed text is returned+ // as-is, with no orphan-tag append. (The original behavior of+ // appending the tag at the trailing edge produced silently+ // malformed XML in ~350 production requests over 9 days.)let blocks = vec![("{{HEADROOM_TAG_0}}".to_string(),"<tag>data</tag>".to_string(),)];- let restored = restore_tags("text without placeholder", &blocks);- assert!(restored.contains("<tag>data</tag>"));+ let compressed = "text without placeholder";+ let restored = restore_tags(compressed, &blocks);+ // Compressed text returned unchanged; original tag NOT injected.+ assert_eq!(restored, compressed);+ assert!(!restored.contains("<tag>"));+ assert!(!restored.contains("</tag>"));+ assert!(!restored.contains("<tag>data</tag>"));+ }++ #[test]+ fn restore_lost_placeholder_idempotent_when_all_missing() {+ // Invariant #3: if every placeholder is missing from the+ // compressed text, the function returns the compressed text+ // byte-for-byte unchanged.+ let blocks = vec![+ ("{{HEADROOM_TAG_0}}".to_string(), "<a>1</a>".to_string()),+ ("{{HEADROOM_TAG_1}}".to_string(), "<b>2</b>".to_string()),+ ("{{HEADROOM_TAG_2}}".to_string(), "<c>3</c>".to_string()),+ ];+ let compressed = "compressor stripped every placeholder";+ let restored = restore_tags(compressed, &blocks);+ assert_eq!(restored, compressed);+ }++ #[test]+ fn restore_partial_loss_keeps_present_drops_lost() {+ // Mixed case: some placeholders survive, others are lost. The+ // surviving ones get substituted; the lost ones are discarded.+ // No orphan-tag bytes appear anywhere in the output.+ let blocks = vec![+ ("{{HEADROOM_TAG_0}}".to_string(), "<a>1</a>".to_string()),+ (+ "{{HEADROOM_TAG_1}}".to_string(),+ "<lost>x</lost>".to_string(),+ ),+ ];+ let compressed = "head {{HEADROOM_TAG_0}} tail";+ let restored = restore_tags(compressed, &blocks);+ assert_eq!(restored, "head <a>1</a> tail");+ assert!(!restored.contains("<lost"));+ assert!(!restored.contains("</lost>"));}#[test]@@ -944,6 +1044,19 @@ mod tests {assert!(blocks.is_empty());}+ #[test]+ fn truncated_close_marker_does_not_panic() {+ // Hotfix-A9: proptest seed `</` would index past end-of-input+ // in `parse_tag_at`. Pre-fix this panicked with an OOB; the+ // bounds-check now returns NotTag and the function falls+ // through to emitting `</` verbatim.+ for text in ["</", "<", "<a/", "<a", "<a /", "</a"] {+ let (cleaned, blocks, _stats) = protect_tags(text, false);+ assert_eq!(cleaned, text);+ assert!(blocks.is_empty());+ }+ }+#[test]fn attribute_with_gt_inside_quotes() {let text = r#"<context attr="a > b">payload</context>"#;@@ -969,4 +1082,191 @@ mod tests {assert_eq!(stats.html_tags_skipped, 1);assert_eq!(stats.orphan_closes, 0);}++ // ─── Hotfix-A9 invariants ────────────────────────────────────────++ /// Count `<custom>` style opening tags (excludes self-closers and+ /// excludes the closing-tag `</…>` form). Any `<` that is followed+ /// by an alphabetic name and ends with `>` (without an embedded+ /// `/>`) counts. Only used by the proptest below — keeps the+ /// invariant check independent of the parser under test.+ fn count_open_tags(s: &str) -> usize {+ let bytes = s.as_bytes();+ let mut count = 0_usize;+ let mut i = 0_usize;+ while i < bytes.len() {+ if bytes[i] != b'<' {+ i += 1;+ continue;+ }+ // Skip closing tags `</…>`.+ if i + 1 < bytes.len() && bytes[i + 1] == b'/' {+ i += 1;+ continue;+ }+ // Must be followed by a name-start char to count as a tag.+ if i + 1 >= bytes.len() || !is_name_start(bytes[i + 1]) {+ i += 1;+ continue;+ }+ // Walk to the matching `>`. If we hit `/>` first, this is+ // self-closing and doesn't count as an unbalanced opener.+ let mut j = i + 1;+ let mut self_closing = false;+ while j < bytes.len() && bytes[j] != b'>' {+ if bytes[j] == b'/' {+ self_closing = true;+ }+ j += 1;+ }+ if j >= bytes.len() {+ // No closing `>` — not a tag.+ break;+ }+ if !self_closing {+ count += 1;+ }+ i = j + 1;+ }+ count+ }++ fn count_close_tags(s: &str) -> usize {+ let bytes = s.as_bytes();+ let mut count = 0_usize;+ let mut i = 0_usize;+ while i < bytes.len() {+ if bytes[i] != b'<' {+ i += 1;+ continue;+ }+ if i + 1 >= bytes.len() || bytes[i + 1] != b'/' {+ i += 1;+ continue;+ }+ // `</name>` — confirm name-start and find the closing `>`.+ if i + 2 >= bytes.len() || !is_name_start(bytes[i + 2]) {+ i += 1;+ continue;+ }+ let mut j = i + 2;+ while j < bytes.len() && bytes[j] != b'>' {+ j += 1;+ }+ if j >= bytes.len() {+ break;+ }+ count += 1;+ i = j + 1;+ }+ count+ }++ proptest::proptest! {+ /// Invariant: `restore_tags` never INTRODUCES tag-count+ /// asymmetry. Concretely: restoring on a compressed text with+ /// any subset of placeholders missing must produce the same+ /// `opens - closes` skew as the cleaned text after stripping+ /// the placeholders. The orphan-append bug fixed by Hotfix-A9+ /// could turn a symmetric `<a>x</a>` into an asymmetric+ /// `compressed-stuff <a>` whenever the placeholder was+ /// dropped — the discard-wrap path makes that impossible+ /// because every protected span is a balanced wrap (or a+ /// self-closer) so dropping it changes opens and closes by+ /// the same amount.+ #[test]+ fn restore_never_introduces_asymmetry(content in "[a-z<>/]{0,200}") {+ let (cleaned, blocks, _stats) = protect_tags(&content, false);+ // Baseline: strip every placeholder from `cleaned`. This+ // is the "lost everything" worst case; the discard-wrap+ // path must produce exactly this output.+ let mut stripped = cleaned.clone();+ for (placeholder, _original) in &blocks {+ stripped = stripped.replace(placeholder.as_str(), "");+ }+ let baseline_skew = count_open_tags(&stripped) as i64+ - count_close_tags(&stripped) as i64;++ // With every placeholder lost, restore_tags must return+ // the compressed text with placeholders dropped — which+ // is exactly `stripped`. So asymmetry equals baseline.+ let restored_all_lost = restore_tags(&stripped, &blocks);+ let lost_skew = count_open_tags(&restored_all_lost) as i64+ - count_close_tags(&restored_all_lost) as i64;+ proptest::prop_assert_eq!(+ lost_skew, baseline_skew,+ "discard-wrap path introduced asymmetry: baseline={}, after_restore={}, restored={:?}",+ baseline_skew, lost_skew, restored_all_lost+ );++ // With every placeholder PRESENT, restore_tags must round-+ // trip exactly to the original `content`, which by+ // construction has the same skew as `content` itself.+ let restored_full = restore_tags(&cleaned, &blocks);+ let full_skew = count_open_tags(&restored_full) as i64+ - count_close_tags(&restored_full) as i64;+ let content_skew = count_open_tags(&content) as i64+ - count_close_tags(&content) as i64;+ proptest::prop_assert_eq!(+ full_skew, content_skew,+ "full-restore path drifted from input skew: input={}, restored={}",+ content_skew, full_skew+ );+ }++ /// Invariant: when every placeholder is stripped before+ /// restore, the function returns the compressed text+ /// byte-for-byte unchanged (no orphan-tag injection, no+ /// whitespace insertion, no prepends/appends).+ #[test]+ fn restore_idempotent_when_all_placeholders_lost(+ content in "[a-z<>/]{0,200}",+ compressed in "[ -~]{0,200}",+ ) {+ let (_cleaned, blocks, _stats) = protect_tags(&content, false);+ // Drop all placeholders by feeding `restore_tags` arbitrary+ // text the compressor "produced". If none of the+ // placeholders happen to appear in `compressed` (the+ // common case for arbitrary strings), the discard-wrap+ // path runs end-to-end.+ let any_placeholder_present = blocks+ .iter()+ .any(|(p, _)| compressed.contains(p.as_str()));+ proptest::prop_assume!(!any_placeholder_present);+ let restored = restore_tags(&compressed, &blocks);+ proptest::prop_assert_eq!(restored, compressed);+ }++ /// Invariant: `restore_tags` never adds bytes that weren't+ /// already in `compressed` or part of a substituted placeholder+ /// original. Concretely: the restored length is at most+ /// `compressed.len()` plus the sum of lengths of originals+ /// that actually got substituted; lost-placeholder originals+ /// contribute zero bytes.+ #[test]+ fn restore_no_orphan_byte_injection(+ content in "[a-z<>/]{0,200}",+ ) {+ let (cleaned, blocks, _stats) = protect_tags(&content, false);+ let restored = restore_tags(&cleaned, &blocks);+ // Sum of the byte-lengths of the originals that were+ // actually substituted (placeholder still present in+ // `cleaned`). Lost placeholders contribute zero.+ let substituted_original_bytes: usize = blocks+ .iter()+ .filter(|(p, _)| cleaned.contains(p.as_str()))+ .map(|(p, original)| original.len().saturating_sub(p.len()))+ .sum();+ // Upper bound: cleaned.len() + delta from substitution.+ // (Substitution replaces each placeholder of len p.len()+ // with original of len original.len(); delta per substitution+ // is original.len() - p.len(), summed across substitutions.)+ let upper_bound = cleaned.len() + substituted_original_bytes;+ proptest::prop_assert!(+ restored.len() <= upper_bound,+ "restored too long: restored.len={} upper_bound={} cleaned.len={}",+ restored.len(), upper_bound, cleaned.len()+ );+ }+ }}diff --git a/headroom/transforms/tag_protector.py b/headroom/transforms/tag_protector.pyindex 9b3a4e2fb..77b6ec907 100644--- a/headroom/transforms/tag_protector.py+++ b/headroom/transforms/tag_protector.py@@ -105,11 +105,20 @@ def restore_tags(protected_blocks: List from :func:`protect_tags`.Returns:- Text with placeholders swapped back to originals. Any- placeholder the compressor accidentally stripped causes the- corresponding original to be appended on the trailing edge of- the output (lossy-compression incident; same fallback the- Python original used).+ Text with placeholders swapped back to originals. If the+ compressor stripped or rewrote a placeholder, the wrap is+ **discarded** — the compressed text is returned as-is for+ that block, and the original tag bytes are NOT re-injected+ anywhere. This is the Hotfix-A9 behavior change vs the+ original "append the orphan tag at the trailing edge"+ fallback, which produced silently malformed XML (an opening+ tag with no closing tag and no body) on production traffic.++ Each lost placeholder emits a structured ERROR-level log+ (``event=tag_protector_placeholder_lost``) so operators can+ alert on the corruption rather than have it disappear into+ a WARN line. Token validation downstream is responsible for+ catching cases where the discard regressed the final output."""return cast("str", _rust_restore_tags(text, protected_blocks))diff --git a/tests/test_tag_protector_invariant.py b/tests/test_tag_protector_invariant.pynew file mode 100644index 000000000..11cc9c956--- /dev/null+++ b/tests/test_tag_protector_invariant.py@@ -0,0 +1,272 @@+"""Hotfix-A9 invariant tests for ``headroom.transforms.tag_protector``.++Pins the discard-wrap semantics end-to-end through the public Python+API. Mirrors the Rust ``proptest`` suite but at the language boundary+so a regression in the PyO3 bridge can't slip past Rust-side tests.++Three invariants:++1. **No orphan-tag injection.** ``restore_tags`` never adds bytes that+ weren't already in ``compressed`` *unless* it is substituting a+ placeholder it found. No appends, no prepends, no whitespace+ inserted outside placeholder substitutions.+2. **Idempotence on missing placeholders.** If every placeholder is+ absent from ``compressed``, ``restore_tags`` returns ``compressed``+ byte-for-byte unchanged.+3. **No introduced asymmetry.** ``restore_tags`` never INTRODUCES+ ``opens != closes`` skew. The orphan-append bug fixed by Hotfix-A9+ could turn a balanced input into an asymmetric output whenever a+ placeholder was lost; the discard-wrap path makes that impossible+ because every protected span is a balanced wrap (or self-closer).++The generator is a small deterministic random walk seeded with a+fixed seed so the suite is reproducible and CI-stable. We don't need+``hypothesis`` to get coverage of the relevant shapes (the alphabet+is `[a-z<>/]` plus a handful of literal tag fragments).+"""++from __future__ import annotations++import random++import pytest++from headroom.transforms.tag_protector import protect_tags, restore_tags++# ──────────────────────────────────────────────────────────────────────+# Helpers — independent open/close counters that don't share code with+# the implementation under test, so a parser bug can't mask itself.+# ──────────────────────────────────────────────────────────────────────+++def _is_name_start(ch: str) -> bool:+ return ch.isalpha() or ch == "_"+++def count_open_tags(s: str) -> int:+ """Count `<name…>` style opening tags. Excludes `</…>` closes and+ `<…/>` self-closers."""+ count = 0+ i = 0+ n = len(s)+ while i < n:+ if s[i] != "<":+ i += 1+ continue+ if i + 1 < n and s[i + 1] == "/":+ i += 1+ continue+ if i + 1 >= n or not _is_name_start(s[i + 1]):+ i += 1+ continue+ # Walk to the matching `>`. Track self-closer marker.+ j = i + 1+ self_closing = False+ while j < n and s[j] != ">":+ if s[j] == "/":+ self_closing = True+ j += 1+ if j >= n:+ break+ if not self_closing:+ count += 1+ i = j + 1+ return count+++def count_close_tags(s: str) -> int:+ """Count `</name>` style closing tags."""+ count = 0+ i = 0+ n = len(s)+ while i < n:+ if s[i] != "<":+ i += 1+ continue+ if i + 1 >= n or s[i + 1] != "/":+ i += 1+ continue+ if i + 2 >= n or not _is_name_start(s[i + 2]):+ i += 1+ continue+ j = i + 2+ while j < n and s[j] != ">":+ j += 1+ if j >= n:+ break+ count += 1+ i = j + 1+ return count+++# ──────────────────────────────────────────────────────────────────────+# Deterministic content generator. Yields a mix of:+# - random ASCII letters/punct+# - balanced custom-tag pairs (`<sys>x</sys>`, `<tool>y</tool>`, …)+# - bare orphan opens / closes / self-closers+# - HTML tags that should not be protected+# Seed is fixed so failures reproduce.+# ──────────────────────────────────────────────────────────────────────++_TAG_NAMES = [+ "sys",+ "tool",+ "thinking",+ "system-reminder",+ "EXTREMELY_IMPORTANT",+ "context",+]+++def _gen_content(rng: random.Random, max_segments: int = 8) -> str:+ parts: list[str] = []+ n_segments = rng.randint(0, max_segments)+ for _ in range(n_segments):+ roll = rng.random()+ if roll < 0.30:+ # Balanced custom tag.+ name = rng.choice(_TAG_NAMES)+ body = "".join(rng.choice("abcde 123") for _ in range(rng.randint(0, 12)))+ parts.append(f"<{name}>{body}</{name}>")+ elif roll < 0.45:+ # Self-closing custom tag.+ name = rng.choice(_TAG_NAMES)+ parts.append(f"<{name}/>")+ elif roll < 0.55:+ # Orphan open (no close) — exercises the asymmetric-input case.+ name = rng.choice(_TAG_NAMES)+ parts.append(f"<{name}>")+ elif roll < 0.65:+ # Orphan close.+ name = rng.choice(_TAG_NAMES)+ parts.append(f"</{name}>")+ elif roll < 0.75:+ # HTML tag — should be passthrough.+ parts.append("<div>plain</div>")+ else:+ # Plain text segment.+ parts.append("".join(rng.choice("abcde <>/") for _ in range(rng.randint(0, 30))))+ return "".join(parts)+++def _strip_placeholders(compressed: str, blocks: list[tuple[str, str]]) -> str:+ out = compressed+ for placeholder, _original in blocks:+ out = out.replace(placeholder, "")+ return out+++# ──────────────────────────────────────────────────────────────────────+# Tests+# ──────────────────────────────────────────────────────────────────────+++# Number of generated cases. Kept moderate so the suite stays fast in+# CI; bug-shrinking is done by the Rust proptest suite which runs the+# same property at the lower layer.+_CASES = 256+_SEED = 0xA9_CAFE++[email protected](scope="module")+def rng() -> random.Random:+ return random.Random(_SEED)+++def test_restore_idempotent_when_all_placeholders_lost(rng: random.Random) -> None:+ """If every placeholder is stripped before restore, the function+ returns the compressed text byte-for-byte unchanged."""+ for _ in range(_CASES):+ content = _gen_content(rng)+ _cleaned, blocks = protect_tags(content)+ if not blocks:+ continue+ # Drop every placeholder by simulating a compressor that ate+ # them. `compressed` carries no placeholders at all.+ compressed = _gen_content(rng)+ # If the random `compressed` happens to contain a placeholder+ # (vanishingly unlikely with our alphabet, but defensive),+ # skip — the property under test is the *no-placeholder* case.+ if any(p in compressed for (p, _o) in blocks):+ continue+ restored = restore_tags(compressed, blocks)+ assert restored == compressed, (+ f"discard-wrap path corrupted output: blocks={blocks}, "+ f"compressed={compressed!r}, restored={restored!r}"+ )+++def test_restore_never_introduces_asymmetry(rng: random.Random) -> None:+ """`restore_tags` never INTRODUCES `opens != closes` skew. With+ every placeholder lost (worst case for the discard-wrap path),+ skew equals the cleaned-minus-placeholders baseline. With every+ placeholder present, skew equals the original input's skew."""+ for _ in range(_CASES):+ content = _gen_content(rng)+ cleaned, blocks = protect_tags(content)++ # Worst case: every placeholder lost. Restored output must+ # equal `cleaned` with placeholders stripped, and so must+ # carry exactly that asymmetry.+ stripped = _strip_placeholders(cleaned, blocks)+ baseline_skew = count_open_tags(stripped) - count_close_tags(stripped)+ restored_lost = restore_tags(stripped, blocks)+ lost_skew = count_open_tags(restored_lost) - count_close_tags(restored_lost)+ assert lost_skew == baseline_skew, (+ f"discard-wrap introduced asymmetry: baseline={baseline_skew}, "+ f"after_restore={lost_skew}, content={content!r}"+ )++ # Full restore: skew matches the original input.+ restored_full = restore_tags(cleaned, blocks)+ full_skew = count_open_tags(restored_full) - count_close_tags(restored_full)+ content_skew = count_open_tags(content) - count_close_tags(content)+ assert full_skew == content_skew, (+ f"full-restore drifted from input skew: input={content_skew}, "+ f"restored={full_skew}, content={content!r}"+ )+++def test_restore_no_orphan_byte_injection(rng: random.Random) -> None:+ """`restore_tags` never adds bytes outside placeholder+ substitution. The restored length is bounded above by the+ compressed length plus the size delta of placeholders that were+ actually substituted; lost-placeholder originals contribute zero+ bytes."""+ for _ in range(_CASES):+ content = _gen_content(rng)+ cleaned, blocks = protect_tags(content)+ restored = restore_tags(cleaned, blocks)+ # Bytes added by substitution: for each placeholder that+ # appears in `cleaned`, replace it with `original` (delta =+ # len(original) - len(placeholder)).+ substituted_delta = sum(+ max(0, len(original) - len(placeholder))+ for placeholder, original in blocks+ if placeholder in cleaned+ )+ upper_bound = len(cleaned) + substituted_delta+ assert len(restored) <= upper_bound, (+ f"restored too long: restored={len(restored)} "+ f"upper_bound={upper_bound} cleaned={len(cleaned)} "+ f"content={content!r}"+ )+++def test_restore_lost_real_world_tag_does_not_inject_orphan() -> None:+ """Anchor-case mirroring the production findings: when the+ placeholder for a `<system-reminder>` block is lost, the+ compressed text must NOT end with an orphan opening tag."""+ blocks = [+ (+ "{{HEADROOM_TAG_0}}",+ "<system-reminder>[Showing lines 1-50 of 1000 total lines]</system-reminder>",+ )+ ]+ compressed = "compressed body without any placeholder reference"+ restored = restore_tags(compressed, blocks)+ # The bug we are killing: opening tag at the END with no body / no close.+ assert not restored.endswith("<system-reminder>")+ assert "<system-reminder>" not in restored+ assert "</system-reminder>" not in restored+ assert restored == compresseddiff --git a/tests/test_transforms/test_tag_protector.py b/tests/test_transforms/test_tag_protector.pyindex cd1bf0f82..0213ddbb4 100644--- a/tests/test_transforms/test_tag_protector.py+++ b/tests/test_transforms/test_tag_protector.py@@ -166,11 +166,45 @@ class TestRestoreTags:assert "<thinking>A</thinking>" in restoredassert "<context>B</context>" in restored- def test_lost_placeholder_appended(self):- """If compression removes a placeholder, the block is appended."""+ def test_lost_placeholder_discards_wrap(self):+ """Hotfix-A9: when compression strips a placeholder, the wrap+ is DISCARDED — the compressed text is returned as-is and the+ original tag bytes are NOT re-injected anywhere. The original+ "append at the trailing edge" fallback produced silently+ malformed XML (orphan opening tag with no closing tag) on+ ~350 production requests over 9 days; that bug is gone."""protected = [("{{HEADROOM_TAG_0}}", "<tag>data</tag>")]- result = restore_tags("text without placeholder", protected)- assert "<tag>data</tag>" in result+ compressed = "text without placeholder"+ result = restore_tags(compressed, protected)+ # Compressed text returned unchanged; original tag NOT injected.+ assert result == compressed+ assert "<tag>" not in result+ assert "</tag>" not in result+ assert "<tag>data</tag>" not in result++ def test_lost_placeholder_idempotent_when_all_missing(self):+ """Invariant: if every placeholder is missing from compressed,+ restore_tags returns compressed byte-for-byte unchanged."""+ protected = [+ ("{{HEADROOM_TAG_0}}", "<a>1</a>"),+ ("{{HEADROOM_TAG_1}}", "<b>2</b>"),+ ("{{HEADROOM_TAG_2}}", "<c>3</c>"),+ ]+ compressed = "compressor stripped every placeholder"+ assert restore_tags(compressed, protected) == compressed++ def test_partial_loss_keeps_present_discards_lost(self):+ """Mixed case: some placeholders survive, others are lost.+ Surviving ones get substituted; lost ones are discarded with+ zero orphan-tag injection."""+ protected = [+ ("{{HEADROOM_TAG_0}}", "<a>1</a>"),+ ("{{HEADROOM_TAG_1}}", "<lost>x</lost>"),+ ]+ result = restore_tags("head {{HEADROOM_TAG_0}} tail", protected)+ assert result == "head <a>1</a> tail"+ assert "<lost" not in result+ assert "</lost>" not in resultdef test_roundtrip_preserves_content(self):original = (