Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix: A9 — tag protector discards wrap on placeholder loss

chopratejas6aacd4805add
diff --git a/crates/headroom-core/proptest-regressions/transforms/tag_protector.txt b/crates/headroom-core/proptest-regressions/transforms/tag_protector.txt
new file mode 100644
index 000000000..760155ae8
--- /dev/null
+++ b/crates/headroom-core/proptest-regressions/transforms/tag_protector.txt
@@ -0,0 +1,9 @@
+# Seeds for failure cases proptest has generated in the past. It is
+# automatically read and these particular cases re-run before any
+# novel cases are generated.
+#
+# It is recommended to check this file in to source control so that
+# everyone who runs the test benefits from these saved cases.
+cc 8fbea33614d948018222b12893746cadfc057a807a80f3e8b63af540614d1eb0 # shrinks to content = "<a>"
+cc e499bfe23a63b8a21d0d8105141c83e83a7f3a1484c19bcf5be0732c7c1bd9c3 # shrinks to content = "</"
+cc 70ba971eb9c85e814763d0af359de0a4a967e8947caf837d4efe751a37292659 # shrinks to content = "</", compressed = ""
diff --git a/crates/headroom-core/src/transforms/tag_protector.rs b/crates/headroom-core/src/transforms/tag_protector.rs
index 2b967b961..2728022b7 100644
--- a/crates/headroom-core/src/transforms/tag_protector.rs
+++ b/crates/headroom-core/src/transforms/tag_protector.rs
@@ -324,6 +324,13 @@ fn parse_tag_at(bytes: &[u8], start: usize) -> TagParse {
if is_close {
i += 1;
}
+ // After consuming a possible '/' we may be at end-of-input
+ // (e.g. literal `</` with nothing after). Guard the bounds
+ // before indexing into `bytes[i]` for the name-start check —
+ // proptest discovered the OOB on input `</`.
+ if i >= n {
+ return TagParse::NotTag;
+ }
let name_start = i;
if !is_name_start(bytes[i]) {
return TagParse::NotTag;
@@ -638,45 +645,94 @@ fn emit_output(
/// Restore protected tag spans after the compressor ran on the
/// cleaned text.
///
+/// # Hotfix-A9 — discard-wrap semantics
+///
/// If a placeholder went missing during compression (the compressor
-/// stripped or rewrote it) the corresponding original is appended to
-/// the output rather than dropped — same fallback semantics as the
-/// Python original. A `tracing::warn!` is emitted at compile-time-
-/// optional verbosity so prod can scrape lossy-compression incidents.
+/// stripped or rewrote it) the wrap is **discarded**: the compressed
+/// text flows downstream as-is and the original tag bytes are NOT
+/// re-injected anywhere. This is a deliberate behavior change vs the
+/// original "append the orphan tag at the trailing edge" fallback,
+/// which produced silently malformed XML (an opening tag with no
+/// closing tag and no body) on ~350 production requests over 9 days.
+///
+/// An ERROR-level `tracing` event with structured fields
+/// (`event = tag_protector_placeholder_lost`, `tag_preview`,
+/// `compressed_length`, optional `request_id`) is emitted per lost
+/// placeholder so operators can alert on the corruption rather than
+/// having it disappear into a WARN line. Token validation downstream
+/// is responsible for catching cases where the discard regressed the
+/// final output vs the original input.
+///
+/// Invariants enforced:
+/// 1. Symmetry — never emit asymmetric tag counts (the input either
+/// survives both opens and closes via successful substitution, or
+/// neither survives via discard).
+/// 2. No orphan tag injection — `restore_tags` adds bytes only as part
+/// of placeholder substitution. No appends, no prepends, no
+/// whitespace insertion outside placeholder substitutions.
+/// 3. Idempotence on missing placeholders — if every placeholder is
+/// absent from `compressed`, the function returns `compressed`
+/// byte-for-byte unchanged.
pub fn restore_tags(text: &str, blocks: &[(String, String)]) -> String {
+ restore_tags_with_request_id(text, blocks, None)
+}
+
+/// Variant of [`restore_tags`] that threads an optional `request_id`
+/// into the structured ERROR log emitted on placeholder loss. The
+/// PyO3 binding currently calls [`restore_tags`] (no request id);
+/// this entry point exists so the proxy layer can wire request
+/// context through once it has one available end-to-end.
+pub fn restore_tags_with_request_id(
+ text: &str,
+ blocks: &[(String, String)],
+ request_id: Option<&str>,
+) -> String {
if blocks.is_empty() {
return text.to_string();
}
let mut result = text.to_string();
- let mut tail_appends: Vec<&str> = Vec::new();
+ let mut lost_count: usize = 0;
+ let compressed_length = text.len();
for (placeholder, original) in blocks {
if result.contains(placeholder.as_str()) {
result = result.replace(placeholder.as_str(), original);
} else {
- tag_lost_warn(original);
- tail_appends.push(original.as_str());
- }
- }
-
- if !tail_appends.is_empty() {
- for original in tail_appends {
- result.push('\n');
- result.push_str(original);
+ lost_count += 1;
+ tag_lost_error(original, compressed_length, request_id);
}
}
-
+ // No tail_appends. The compressed text is returned with the
+ // wraps for the lost placeholders fully discarded — never
+ // appended back as orphan opens (Hotfix-A9).
+ let _ = lost_count; // surfaced via the per-event log; aggregate
+ // counters live on the stats sidecar in
+ // callers that have one.
result
}
#[inline(never)]
-fn tag_lost_warn(original: &str) {
+fn tag_lost_error(original: &str, compressed_length: usize, request_id: Option<&str>) {
let preview: String = original.chars().take(80).collect();
- tracing::warn!(
- target: "headroom::tag_protector",
- preview = %preview,
- "tag placeholder lost during compression, appending original"
- );
+ match request_id {
+ Some(rid) => tracing::error!(
+ target: "headroom::tag_protector",
+ event = "tag_protector_placeholder_lost",
+ tag_preview = %preview,
+ compressed_length = compressed_length,
+ request_id = %rid,
+ action = "discarded_wrap",
+ "tag placeholder lost during compression — wrap discarded"
+ ),
+ None => tracing::error!(
+ target: "headroom::tag_protector",
+ event = "tag_protector_placeholder_lost",
+ tag_preview = %preview,
+ compressed_length = compressed_length,
+ action = "discarded_wrap",
+ "tag placeholder lost during compression — wrap discarded"
+ ),
+ }
}
// ─── Tiny byte-search helper ──────────────────────────────────────────
@@ -824,13 +880,57 @@ mod tests {
}
#[test]
- fn restore_lost_placeholder_appended() {
+ fn restore_lost_placeholder_discards_wrap() {
+ // Hotfix-A9: when a placeholder is missing from the compressed
+ // text, the wrap is DISCARDED — the compressed text is returned
+ // as-is, with no orphan-tag append. (The original behavior of
+ // appending the tag at the trailing edge produced silently
+ // malformed XML in ~350 production requests over 9 days.)
let blocks = vec![(
"{{HEADROOM_TAG_0}}".to_string(),
"<tag>data</tag>".to_string(),
)];
- let restored = restore_tags("text without placeholder", &blocks);
- assert!(restored.contains("<tag>data</tag>"));
+ let compressed = "text without placeholder";
+ let restored = restore_tags(compressed, &blocks);
+ // Compressed text returned unchanged; original tag NOT injected.
+ assert_eq!(restored, compressed);
+ assert!(!restored.contains("<tag>"));
+ assert!(!restored.contains("</tag>"));
+ assert!(!restored.contains("<tag>data</tag>"));
+ }
+
+ #[test]
+ fn restore_lost_placeholder_idempotent_when_all_missing() {
+ // Invariant #3: if every placeholder is missing from the
+ // compressed text, the function returns the compressed text
+ // byte-for-byte unchanged.
+ let blocks = vec![
+ ("{{HEADROOM_TAG_0}}".to_string(), "<a>1</a>".to_string()),
+ ("{{HEADROOM_TAG_1}}".to_string(), "<b>2</b>".to_string()),
+ ("{{HEADROOM_TAG_2}}".to_string(), "<c>3</c>".to_string()),
+ ];
+ let compressed = "compressor stripped every placeholder";
+ let restored = restore_tags(compressed, &blocks);
+ assert_eq!(restored, compressed);
+ }
+
+ #[test]
+ fn restore_partial_loss_keeps_present_drops_lost() {
+ // Mixed case: some placeholders survive, others are lost. The
+ // surviving ones get substituted; the lost ones are discarded.
+ // No orphan-tag bytes appear anywhere in the output.
+ let blocks = vec![
+ ("{{HEADROOM_TAG_0}}".to_string(), "<a>1</a>".to_string()),
+ (
+ "{{HEADROOM_TAG_1}}".to_string(),
+ "<lost>x</lost>".to_string(),
+ ),
+ ];
+ let compressed = "head {{HEADROOM_TAG_0}} tail";
+ let restored = restore_tags(compressed, &blocks);
+ assert_eq!(restored, "head <a>1</a> tail");
+ assert!(!restored.contains("<lost"));
+ assert!(!restored.contains("</lost>"));
}
#[test]
@@ -944,6 +1044,19 @@ mod tests {
assert!(blocks.is_empty());
}
+ #[test]
+ fn truncated_close_marker_does_not_panic() {
+ // Hotfix-A9: proptest seed `</` would index past end-of-input
+ // in `parse_tag_at`. Pre-fix this panicked with an OOB; the
+ // bounds-check now returns NotTag and the function falls
+ // through to emitting `</` verbatim.
+ for text in ["</", "<", "<a/", "<a", "<a /", "</a"] {
+ let (cleaned, blocks, _stats) = protect_tags(text, false);
+ assert_eq!(cleaned, text);
+ assert!(blocks.is_empty());
+ }
+ }
+
#[test]
fn attribute_with_gt_inside_quotes() {
let text = r#"<context attr="a > b">payload</context>"#;
@@ -969,4 +1082,191 @@ mod tests {
assert_eq!(stats.html_tags_skipped, 1);
assert_eq!(stats.orphan_closes, 0);
}
+
+ // ─── Hotfix-A9 invariants ────────────────────────────────────────
+
+ /// Count `<custom>` style opening tags (excludes self-closers and
+ /// excludes the closing-tag `</…>` form). Any `<` that is followed
+ /// by an alphabetic name and ends with `>` (without an embedded
+ /// `/>`) counts. Only used by the proptest below — keeps the
+ /// invariant check independent of the parser under test.
+ fn count_open_tags(s: &str) -> usize {
+ let bytes = s.as_bytes();
+ let mut count = 0_usize;
+ let mut i = 0_usize;
+ while i < bytes.len() {
+ if bytes[i] != b'<' {
+ i += 1;
+ continue;
+ }
+ // Skip closing tags `</…>`.
+ if i + 1 < bytes.len() && bytes[i + 1] == b'/' {
+ i += 1;
+ continue;
+ }
+ // Must be followed by a name-start char to count as a tag.
+ if i + 1 >= bytes.len() || !is_name_start(bytes[i + 1]) {
+ i += 1;
+ continue;
+ }
+ // Walk to the matching `>`. If we hit `/>` first, this is
+ // self-closing and doesn't count as an unbalanced opener.
+ let mut j = i + 1;
+ let mut self_closing = false;
+ while j < bytes.len() && bytes[j] != b'>' {
+ if bytes[j] == b'/' {
+ self_closing = true;
+ }
+ j += 1;
+ }
+ if j >= bytes.len() {
+ // No closing `>` — not a tag.
+ break;
+ }
+ if !self_closing {
+ count += 1;
+ }
+ i = j + 1;
+ }
+ count
+ }
+
+ fn count_close_tags(s: &str) -> usize {
+ let bytes = s.as_bytes();
+ let mut count = 0_usize;
+ let mut i = 0_usize;
+ while i < bytes.len() {
+ if bytes[i] != b'<' {
+ i += 1;
+ continue;
+ }
+ if i + 1 >= bytes.len() || bytes[i + 1] != b'/' {
+ i += 1;
+ continue;
+ }
+ // `</name>` — confirm name-start and find the closing `>`.
+ if i + 2 >= bytes.len() || !is_name_start(bytes[i + 2]) {
+ i += 1;
+ continue;
+ }
+ let mut j = i + 2;
+ while j < bytes.len() && bytes[j] != b'>' {
+ j += 1;
+ }
+ if j >= bytes.len() {
+ break;
+ }
+ count += 1;
+ i = j + 1;
+ }
+ count
+ }
+
+ proptest::proptest! {
+ /// Invariant: `restore_tags` never INTRODUCES tag-count
+ /// asymmetry. Concretely: restoring on a compressed text with
+ /// any subset of placeholders missing must produce the same
+ /// `opens - closes` skew as the cleaned text after stripping
+ /// the placeholders. The orphan-append bug fixed by Hotfix-A9
+ /// could turn a symmetric `<a>x</a>` into an asymmetric
+ /// `compressed-stuff <a>` whenever the placeholder was
+ /// dropped — the discard-wrap path makes that impossible
+ /// because every protected span is a balanced wrap (or a
+ /// self-closer) so dropping it changes opens and closes by
+ /// the same amount.
+ #[test]
+ fn restore_never_introduces_asymmetry(content in "[a-z<>/]{0,200}") {
+ let (cleaned, blocks, _stats) = protect_tags(&content, false);
+ // Baseline: strip every placeholder from `cleaned`. This
+ // is the "lost everything" worst case; the discard-wrap
+ // path must produce exactly this output.
+ let mut stripped = cleaned.clone();
+ for (placeholder, _original) in &blocks {
+ stripped = stripped.replace(placeholder.as_str(), "");
+ }
+ let baseline_skew = count_open_tags(&stripped) as i64
+ - count_close_tags(&stripped) as i64;
+
+ // With every placeholder lost, restore_tags must return
+ // the compressed text with placeholders dropped — which
+ // is exactly `stripped`. So asymmetry equals baseline.
+ let restored_all_lost = restore_tags(&stripped, &blocks);
+ let lost_skew = count_open_tags(&restored_all_lost) as i64
+ - count_close_tags(&restored_all_lost) as i64;
+ proptest::prop_assert_eq!(
+ lost_skew, baseline_skew,
+ "discard-wrap path introduced asymmetry: baseline={}, after_restore={}, restored={:?}",
+ baseline_skew, lost_skew, restored_all_lost
+ );
+
+ // With every placeholder PRESENT, restore_tags must round-
+ // trip exactly to the original `content`, which by
+ // construction has the same skew as `content` itself.
+ let restored_full = restore_tags(&cleaned, &blocks);
+ let full_skew = count_open_tags(&restored_full) as i64
+ - count_close_tags(&restored_full) as i64;
+ let content_skew = count_open_tags(&content) as i64
+ - count_close_tags(&content) as i64;
+ proptest::prop_assert_eq!(
+ full_skew, content_skew,
+ "full-restore path drifted from input skew: input={}, restored={}",
+ content_skew, full_skew
+ );
+ }
+
+ /// Invariant: when every placeholder is stripped before
+ /// restore, the function returns the compressed text
+ /// byte-for-byte unchanged (no orphan-tag injection, no
+ /// whitespace insertion, no prepends/appends).
+ #[test]
+ fn restore_idempotent_when_all_placeholders_lost(
+ content in "[a-z<>/]{0,200}",
+ compressed in "[ -~]{0,200}",
+ ) {
+ let (_cleaned, blocks, _stats) = protect_tags(&content, false);
+ // Drop all placeholders by feeding `restore_tags` arbitrary
+ // text the compressor "produced". If none of the
+ // placeholders happen to appear in `compressed` (the
+ // common case for arbitrary strings), the discard-wrap
+ // path runs end-to-end.
+ let any_placeholder_present = blocks
+ .iter()
+ .any(|(p, _)| compressed.contains(p.as_str()));
+ proptest::prop_assume!(!any_placeholder_present);
+ let restored = restore_tags(&compressed, &blocks);
+ proptest::prop_assert_eq!(restored, compressed);
+ }
+
+ /// Invariant: `restore_tags` never adds bytes that weren't
+ /// already in `compressed` or part of a substituted placeholder
+ /// original. Concretely: the restored length is at most
+ /// `compressed.len()` plus the sum of lengths of originals
+ /// that actually got substituted; lost-placeholder originals
+ /// contribute zero bytes.
+ #[test]
+ fn restore_no_orphan_byte_injection(
+ content in "[a-z<>/]{0,200}",
+ ) {
+ let (cleaned, blocks, _stats) = protect_tags(&content, false);
+ let restored = restore_tags(&cleaned, &blocks);
+ // Sum of the byte-lengths of the originals that were
+ // actually substituted (placeholder still present in
+ // `cleaned`). Lost placeholders contribute zero.
+ let substituted_original_bytes: usize = blocks
+ .iter()
+ .filter(|(p, _)| cleaned.contains(p.as_str()))
+ .map(|(p, original)| original.len().saturating_sub(p.len()))
+ .sum();
+ // Upper bound: cleaned.len() + delta from substitution.
+ // (Substitution replaces each placeholder of len p.len()
+ // with original of len original.len(); delta per substitution
+ // is original.len() - p.len(), summed across substitutions.)
+ let upper_bound = cleaned.len() + substituted_original_bytes;
+ proptest::prop_assert!(
+ restored.len() <= upper_bound,
+ "restored too long: restored.len={} upper_bound={} cleaned.len={}",
+ restored.len(), upper_bound, cleaned.len()
+ );
+ }
+ }
}
diff --git a/headroom/transforms/tag_protector.py b/headroom/transforms/tag_protector.py
index 9b3a4e2fb..77b6ec907 100644
--- a/headroom/transforms/tag_protector.py
+++ b/headroom/transforms/tag_protector.py
@@ -105,11 +105,20 @@ def restore_tags(
protected_blocks: List from :func:`protect_tags`.
Returns:
- Text with placeholders swapped back to originals. Any
- placeholder the compressor accidentally stripped causes the
- corresponding original to be appended on the trailing edge of
- the output (lossy-compression incident; same fallback the
- Python original used).
+ Text with placeholders swapped back to originals. If the
+ compressor stripped or rewrote a placeholder, the wrap is
+ **discarded** — the compressed text is returned as-is for
+ that block, and the original tag bytes are NOT re-injected
+ anywhere. This is the Hotfix-A9 behavior change vs the
+ original "append the orphan tag at the trailing edge"
+ fallback, which produced silently malformed XML (an opening
+ tag with no closing tag and no body) on production traffic.
+
+ Each lost placeholder emits a structured ERROR-level log
+ (``event=tag_protector_placeholder_lost``) so operators can
+ alert on the corruption rather than have it disappear into
+ a WARN line. Token validation downstream is responsible for
+ catching cases where the discard regressed the final output.
"""
return cast("str", _rust_restore_tags(text, protected_blocks))
diff --git a/tests/test_tag_protector_invariant.py b/tests/test_tag_protector_invariant.py
new file mode 100644
index 000000000..11cc9c956
--- /dev/null
+++ b/tests/test_tag_protector_invariant.py
@@ -0,0 +1,272 @@
+"""Hotfix-A9 invariant tests for ``headroom.transforms.tag_protector``.
+
+Pins the discard-wrap semantics end-to-end through the public Python
+API. Mirrors the Rust ``proptest`` suite but at the language boundary
+so a regression in the PyO3 bridge can't slip past Rust-side tests.
+
+Three invariants:
+
+1. **No orphan-tag injection.** ``restore_tags`` never adds bytes that
+ weren't already in ``compressed`` *unless* it is substituting a
+ placeholder it found. No appends, no prepends, no whitespace
+ inserted outside placeholder substitutions.
+2. **Idempotence on missing placeholders.** If every placeholder is
+ absent from ``compressed``, ``restore_tags`` returns ``compressed``
+ byte-for-byte unchanged.
+3. **No introduced asymmetry.** ``restore_tags`` never INTRODUCES
+ ``opens != closes`` skew. The orphan-append bug fixed by Hotfix-A9
+ could turn a balanced input into an asymmetric output whenever a
+ placeholder was lost; the discard-wrap path makes that impossible
+ because every protected span is a balanced wrap (or self-closer).
+
+The generator is a small deterministic random walk seeded with a
+fixed seed so the suite is reproducible and CI-stable. We don't need
+``hypothesis`` to get coverage of the relevant shapes (the alphabet
+is `[a-z<>/]` plus a handful of literal tag fragments).
+"""
+
+from __future__ import annotations
+
+import random
+
+import pytest
+
+from headroom.transforms.tag_protector import protect_tags, restore_tags
+
+# ──────────────────────────────────────────────────────────────────────
+# Helpers — independent open/close counters that don't share code with
+# the implementation under test, so a parser bug can't mask itself.
+# ──────────────────────────────────────────────────────────────────────
+
+
+def _is_name_start(ch: str) -> bool:
+ return ch.isalpha() or ch == "_"
+
+
+def count_open_tags(s: str) -> int:
+ """Count `<name…>` style opening tags. Excludes `</…>` closes and
+ `<…/>` self-closers."""
+ count = 0
+ i = 0
+ n = len(s)
+ while i < n:
+ if s[i] != "<":
+ i += 1
+ continue
+ if i + 1 < n and s[i + 1] == "/":
+ i += 1
+ continue
+ if i + 1 >= n or not _is_name_start(s[i + 1]):
+ i += 1
+ continue
+ # Walk to the matching `>`. Track self-closer marker.
+ j = i + 1
+ self_closing = False
+ while j < n and s[j] != ">":
+ if s[j] == "/":
+ self_closing = True
+ j += 1
+ if j >= n:
+ break
+ if not self_closing:
+ count += 1
+ i = j + 1
+ return count
+
+
+def count_close_tags(s: str) -> int:
+ """Count `</name>` style closing tags."""
+ count = 0
+ i = 0
+ n = len(s)
+ while i < n:
+ if s[i] != "<":
+ i += 1
+ continue
+ if i + 1 >= n or s[i + 1] != "/":
+ i += 1
+ continue
+ if i + 2 >= n or not _is_name_start(s[i + 2]):
+ i += 1
+ continue
+ j = i + 2
+ while j < n and s[j] != ">":
+ j += 1
+ if j >= n:
+ break
+ count += 1
+ i = j + 1
+ return count
+
+
+# ──────────────────────────────────────────────────────────────────────
+# Deterministic content generator. Yields a mix of:
+# - random ASCII letters/punct
+# - balanced custom-tag pairs (`<sys>x</sys>`, `<tool>y</tool>`, …)
+# - bare orphan opens / closes / self-closers
+# - HTML tags that should not be protected
+# Seed is fixed so failures reproduce.
+# ──────────────────────────────────────────────────────────────────────
+
+_TAG_NAMES = [
+ "sys",
+ "tool",
+ "thinking",
+ "system-reminder",
+ "EXTREMELY_IMPORTANT",
+ "context",
+]
+
+
+def _gen_content(rng: random.Random, max_segments: int = 8) -> str:
+ parts: list[str] = []
+ n_segments = rng.randint(0, max_segments)
+ for _ in range(n_segments):
+ roll = rng.random()
+ if roll < 0.30:
+ # Balanced custom tag.
+ name = rng.choice(_TAG_NAMES)
+ body = "".join(rng.choice("abcde 123") for _ in range(rng.randint(0, 12)))
+ parts.append(f"<{name}>{body}</{name}>")
+ elif roll < 0.45:
+ # Self-closing custom tag.
+ name = rng.choice(_TAG_NAMES)
+ parts.append(f"<{name}/>")
+ elif roll < 0.55:
+ # Orphan open (no close) — exercises the asymmetric-input case.
+ name = rng.choice(_TAG_NAMES)
+ parts.append(f"<{name}>")
+ elif roll < 0.65:
+ # Orphan close.
+ name = rng.choice(_TAG_NAMES)
+ parts.append(f"</{name}>")
+ elif roll < 0.75:
+ # HTML tag — should be passthrough.
+ parts.append("<div>plain</div>")
+ else:
+ # Plain text segment.
+ parts.append("".join(rng.choice("abcde <>/") for _ in range(rng.randint(0, 30))))
+ return "".join(parts)
+
+
+def _strip_placeholders(compressed: str, blocks: list[tuple[str, str]]) -> str:
+ out = compressed
+ for placeholder, _original in blocks:
+ out = out.replace(placeholder, "")
+ return out
+
+
+# ──────────────────────────────────────────────────────────────────────
+# Tests
+# ──────────────────────────────────────────────────────────────────────
+
+
+# Number of generated cases. Kept moderate so the suite stays fast in
+# CI; bug-shrinking is done by the Rust proptest suite which runs the
+# same property at the lower layer.
+_CASES = 256
+_SEED = 0xA9_CAFE
+
+
[email protected](scope="module")
+def rng() -> random.Random:
+ return random.Random(_SEED)
+
+
+def test_restore_idempotent_when_all_placeholders_lost(rng: random.Random) -> None:
+ """If every placeholder is stripped before restore, the function
+ returns the compressed text byte-for-byte unchanged."""
+ for _ in range(_CASES):
+ content = _gen_content(rng)
+ _cleaned, blocks = protect_tags(content)
+ if not blocks:
+ continue
+ # Drop every placeholder by simulating a compressor that ate
+ # them. `compressed` carries no placeholders at all.
+ compressed = _gen_content(rng)
+ # If the random `compressed` happens to contain a placeholder
+ # (vanishingly unlikely with our alphabet, but defensive),
+ # skip — the property under test is the *no-placeholder* case.
+ if any(p in compressed for (p, _o) in blocks):
+ continue
+ restored = restore_tags(compressed, blocks)
+ assert restored == compressed, (
+ f"discard-wrap path corrupted output: blocks={blocks}, "
+ f"compressed={compressed!r}, restored={restored!r}"
+ )
+
+
+def test_restore_never_introduces_asymmetry(rng: random.Random) -> None:
+ """`restore_tags` never INTRODUCES `opens != closes` skew. With
+ every placeholder lost (worst case for the discard-wrap path),
+ skew equals the cleaned-minus-placeholders baseline. With every
+ placeholder present, skew equals the original input's skew."""
+ for _ in range(_CASES):
+ content = _gen_content(rng)
+ cleaned, blocks = protect_tags(content)
+
+ # Worst case: every placeholder lost. Restored output must
+ # equal `cleaned` with placeholders stripped, and so must
+ # carry exactly that asymmetry.
+ stripped = _strip_placeholders(cleaned, blocks)
+ baseline_skew = count_open_tags(stripped) - count_close_tags(stripped)
+ restored_lost = restore_tags(stripped, blocks)
+ lost_skew = count_open_tags(restored_lost) - count_close_tags(restored_lost)
+ assert lost_skew == baseline_skew, (
+ f"discard-wrap introduced asymmetry: baseline={baseline_skew}, "
+ f"after_restore={lost_skew}, content={content!r}"
+ )
+
+ # Full restore: skew matches the original input.
+ restored_full = restore_tags(cleaned, blocks)
+ full_skew = count_open_tags(restored_full) - count_close_tags(restored_full)
+ content_skew = count_open_tags(content) - count_close_tags(content)
+ assert full_skew == content_skew, (
+ f"full-restore drifted from input skew: input={content_skew}, "
+ f"restored={full_skew}, content={content!r}"
+ )
+
+
+def test_restore_no_orphan_byte_injection(rng: random.Random) -> None:
+ """`restore_tags` never adds bytes outside placeholder
+ substitution. The restored length is bounded above by the
+ compressed length plus the size delta of placeholders that were
+ actually substituted; lost-placeholder originals contribute zero
+ bytes."""
+ for _ in range(_CASES):
+ content = _gen_content(rng)
+ cleaned, blocks = protect_tags(content)
+ restored = restore_tags(cleaned, blocks)
+ # Bytes added by substitution: for each placeholder that
+ # appears in `cleaned`, replace it with `original` (delta =
+ # len(original) - len(placeholder)).
+ substituted_delta = sum(
+ max(0, len(original) - len(placeholder))
+ for placeholder, original in blocks
+ if placeholder in cleaned
+ )
+ upper_bound = len(cleaned) + substituted_delta
+ assert len(restored) <= upper_bound, (
+ f"restored too long: restored={len(restored)} "
+ f"upper_bound={upper_bound} cleaned={len(cleaned)} "
+ f"content={content!r}"
+ )
+
+
+def test_restore_lost_real_world_tag_does_not_inject_orphan() -> None:
+ """Anchor-case mirroring the production findings: when the
+ placeholder for a `<system-reminder>` block is lost, the
+ compressed text must NOT end with an orphan opening tag."""
+ blocks = [
+ (
+ "{{HEADROOM_TAG_0}}",
+ "<system-reminder>[Showing lines 1-50 of 1000 total lines]</system-reminder>",
+ )
+ ]
+ compressed = "compressed body without any placeholder reference"
+ restored = restore_tags(compressed, blocks)
+ # The bug we are killing: opening tag at the END with no body / no close.
+ assert not restored.endswith("<system-reminder>")
+ assert "<system-reminder>" not in restored
+ assert "</system-reminder>" not in restored
+ assert restored == compressed
diff --git a/tests/test_transforms/test_tag_protector.py b/tests/test_transforms/test_tag_protector.py
index cd1bf0f82..0213ddbb4 100644
--- a/tests/test_transforms/test_tag_protector.py
+++ b/tests/test_transforms/test_tag_protector.py
@@ -166,11 +166,45 @@ class TestRestoreTags:
assert "<thinking>A</thinking>" in restored
assert "<context>B</context>" in restored
- def test_lost_placeholder_appended(self):
- """If compression removes a placeholder, the block is appended."""
+ def test_lost_placeholder_discards_wrap(self):
+ """Hotfix-A9: when compression strips a placeholder, the wrap
+ is DISCARDED — the compressed text is returned as-is and the
+ original tag bytes are NOT re-injected anywhere. The original
+ "append at the trailing edge" fallback produced silently
+ malformed XML (orphan opening tag with no closing tag) on
+ ~350 production requests over 9 days; that bug is gone."""
protected = [("{{HEADROOM_TAG_0}}", "<tag>data</tag>")]
- result = restore_tags("text without placeholder", protected)
- assert "<tag>data</tag>" in result
+ compressed = "text without placeholder"
+ result = restore_tags(compressed, protected)
+ # Compressed text returned unchanged; original tag NOT injected.
+ assert result == compressed
+ assert "<tag>" not in result
+ assert "</tag>" not in result
+ assert "<tag>data</tag>" not in result
+
+ def test_lost_placeholder_idempotent_when_all_missing(self):
+ """Invariant: if every placeholder is missing from compressed,
+ restore_tags returns compressed byte-for-byte unchanged."""
+ protected = [
+ ("{{HEADROOM_TAG_0}}", "<a>1</a>"),
+ ("{{HEADROOM_TAG_1}}", "<b>2</b>"),
+ ("{{HEADROOM_TAG_2}}", "<c>3</c>"),
+ ]
+ compressed = "compressor stripped every placeholder"
+ assert restore_tags(compressed, protected) == compressed
+
+ def test_partial_loss_keeps_present_discards_lost(self):
+ """Mixed case: some placeholders survive, others are lost.
+ Surviving ones get substituted; lost ones are discarded with
+ zero orphan-tag injection."""
+ protected = [
+ ("{{HEADROOM_TAG_0}}", "<a>1</a>"),
+ ("{{HEADROOM_TAG_1}}", "<lost>x</lost>"),
+ ]
+ result = restore_tags("head {{HEADROOM_TAG_0}} tail", protected)
+ assert result == "head <a>1</a> tail"
+ assert "<lost" not in result
+ assert "</lost>" not in result
def test_roundtrip_preserves_content(self):
original = (