Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(observability): G3 remediation — bound cardinality + wire dead metrics
chopratejas2a717a993ee9
diff --git a/crates/headroom-proxy/Cargo.toml b/crates/headroom-proxy/Cargo.tomlindex 88690ce50..d2c889d86 100644--- a/crates/headroom-proxy/Cargo.toml+++ b/crates/headroom-proxy/Cargo.toml@@ -58,7 +58,16 @@ crc32fast = "1"# don't need (we serve text-format scrapes only), so we disable# defaults and re-enable nothing — pure registry + counter +# histogram + text encoder is sufficient.-prometheus = { version = "0.13", default-features = false }+#+# H4 fix: the H3 force-zero contract (in+# `observability::prometheus::handle_metrics`) relies on this+# crate's v0.13 `gather()` semantics — empty MetricVec families are+# omitted from the scrape, so we force-touch each counter / gauge+# with a sentinel label to surface HELP/TYPE on boot. Pinning the+# exact patch version (no caret, no `~`) so a future minor-version+# bump cannot silently change the alarm contract; the bump must be+# an explicit code review that re-validates the contract.+prometheus = { version = "=0.13.4", default-features = false }# PR-E6: SHA-256 over canonical bytes of the cache hot zone (system,# tools, early messages) for cache-bust drift detection. Already in# the dev-dependencies (and pulled transitively by `aws-sigv4` viadiff --git a/crates/headroom-proxy/src/compression/live_zone_anthropic.rs b/crates/headroom-proxy/src/compression/live_zone_anthropic.rsindex 4d3dd7e75..c52b6f078 100644--- a/crates/headroom-proxy/src/compression/live_zone_anthropic.rs+++ b/crates/headroom-proxy/src/compression/live_zone_anthropic.rs@@ -55,6 +55,22 @@ use crate::cache_stabilization::tool_def_normalize::{use crate::compression::resolve_frozen_count;use crate::config::{CacheControlAutoFrozen, CompressionMode};+/// Per-strategy aggregate token counts for the+/// `proxy_compression_ratio_by_strategy` metric. One entry per+/// distinct `strategy` tag observed in the manifest's+/// `BlockAction::Compressed` blocks. `H1` remediation: the proxy+/// previously emitted the same aggregate ratio per strategy when+/// multiple strategies ran on one body — meaning Phase H per-+/// strategy dashboards read garbage. This struct surfaces the+/// genuine per-strategy values so the emit loop reports the right+/// numbers.+#[derive(Debug, Clone, Copy)]+pub struct PerStrategyTokens {+ pub strategy: &'static str,+ pub original_tokens: usize,+ pub compressed_tokens: usize,+}+/// What happened. The caller uses the variant to decide whether to/// forward the original bytes (everything PR-B2 lands on) or a/// modified body (PR-B3+).@@ -72,6 +88,13 @@ pub enum Outcome {tokens_after: usize,strategies_applied: Vec<&'static str>,markers_inserted: Vec<String>,+ /// H1 remediation: per-strategy `(before, after)` aggregate+ /// for the `proxy_compression_ratio_by_strategy` metric.+ /// Empty when the proxy compressed via a non-block path+ /// (e.g. Phase E normalization that doesn't have per-strategy+ /// token accounting) — emit-site falls back to one+ /// aggregate-labelled sample.+ per_strategy_tokens: Vec<PerStrategyTokens>,},/// Dispatcher opted out for a reason we can name.Passthrough { reason: PassthroughReason },@@ -379,6 +402,12 @@ pub fn compress_anthropic_request(tokens_after: 0,strategies_applied: strategies,markers_inserted: e3_locations,+ // H1 remediation: Phase E normalization passes+ // (E1 sort, E3 cache_control auto-placement)+ // mutate bytes but don't have per-strategy+ // token accounting. Empty vec → emit-site falls+ // back to one aggregate sample.+ per_strategy_tokens: Vec::new(),}} else {Outcome::NoCompression@@ -390,27 +419,63 @@ pub fn compress_anthropic_request(// dispatcher used to gate per-block acceptance — so the// saving the proxy logs is the saving the cache will// actually see.+ //+ // H1 + C5 remediation:+ // - Per-strategy `(before, after)` aggregate populated+ // from the manifest so the proxy emits one+ // `proxy_compression_ratio_by_strategy` sample with+ // the right numbers per strategy (instead of the+ // same aggregate ratio per strategy, which was the+ // pre-fix behavior).+ // - Every `BlockAction::RejectedNotSmaller` increments+ // the `proxy_compression_rejected_by_token_check_total`+ // counter so dashboards can attribute "compressor ran+ // but kept the original" cases.let mut original_bytes_total: usize = 0;let mut compressed_bytes_total: usize = 0;let mut original_tokens_total: usize = 0;let mut compressed_tokens_total: usize = 0;let mut strategies: Vec<&'static str> = Vec::new();+ let mut per_strategy_tokens: Vec<PerStrategyTokens> = Vec::new();for entry in &manifest.block_outcomes {- if let BlockAction::Compressed {- strategy,- original_bytes,- compressed_bytes,- original_tokens,- compressed_tokens,- } = entry.action- {- original_bytes_total += original_bytes;- compressed_bytes_total += compressed_bytes;- original_tokens_total += original_tokens;- compressed_tokens_total += compressed_tokens;- if !strategies.contains(&strategy) {- strategies.push(strategy);+ match entry.action {+ BlockAction::Compressed {+ strategy,+ original_bytes,+ compressed_bytes,+ original_tokens,+ compressed_tokens,+ } => {+ original_bytes_total += original_bytes;+ compressed_bytes_total += compressed_bytes;+ original_tokens_total += original_tokens;+ compressed_tokens_total += compressed_tokens;+ if !strategies.contains(&strategy) {+ strategies.push(strategy);+ }+ // H1: accumulate per-strategy tokens (one+ // entry per strategy; multiple blocks of+ // the same strategy sum).+ if let Some(slot) = per_strategy_tokens+ .iter_mut()+ .find(|s| s.strategy == strategy)+ {+ slot.original_tokens += original_tokens;+ slot.compressed_tokens += compressed_tokens;+ } else {+ per_strategy_tokens.push(PerStrategyTokens {+ strategy,+ original_tokens,+ compressed_tokens,+ });+ }+ }+ BlockAction::RejectedNotSmaller { strategy, .. } => {+ // C5: surface the tokenizer-validated+ // rejection in the dedicated counter.+ crate::observability::record_compression_rejected_by_token_check(strategy);}+ _ => {}}}// Stitch in the PR-E1 / PR-E2 / PR-E3 strategy tags so@@ -463,6 +528,7 @@ pub fn compress_anthropic_request(// PR-E3 surfaces tool-slot location(s); PR-B7 will// append CCR retrieval markers when wired.markers_inserted: e3_locations,+ per_strategy_tokens,}}Err(LiveZoneError::BodyNotJson(_)) => {diff --git a/crates/headroom-proxy/src/compression/live_zone_openai.rs b/crates/headroom-proxy/src/compression/live_zone_openai.rsindex 23271022c..4bdec6571 100644--- a/crates/headroom-proxy/src/compression/live_zone_openai.rs+++ b/crates/headroom-proxy/src/compression/live_zone_openai.rs@@ -39,7 +39,7 @@ use serde_json::Value;use crate::cache_stabilization::tool_def_normalize::{any_tool_has_cache_control, sort_schema_keys_recursive, sort_tools_deterministically,};-use crate::compression::{Outcome, PassthroughReason};+use crate::compression::{Outcome, PassthroughReason, PerStrategyTokens};use crate::config::CompressionMode;/// OpenAI Chat Completions live-zone compression entry point.@@ -159,6 +159,7 @@ pub fn compress_openai_chat_request(tokens_after: 0,strategies_applied: normalization_applied.strategies(),markers_inserted: Vec::new(),+ per_strategy_tokens: Vec::new(),};}Outcome::NoCompression@@ -167,11 +168,17 @@ pub fn compress_openai_chat_request(// Aggregate manifest stats. Mirrors the Anthropic// module — same metric shape so dashboards don't need// to special-case the provider.+ //+ // H1 + C5 remediation: per-strategy token accumulation+ // for the proxy's per-strategy compression-ratio metric ++ // every rejected-not-smaller block bumps the dedicated+ // counter.let mut original_bytes_total: usize = 0;let mut compressed_bytes_total: usize = 0;let mut original_tokens_total: usize = 0;let mut compressed_tokens_total: usize = 0;let mut strategies: Vec<&'static str> = Vec::new();+ let mut per_strategy_tokens: Vec<PerStrategyTokens> = Vec::new();let mut had_compressor_error = false;for entry in &manifest.block_outcomes {match entry.action {@@ -189,6 +196,22 @@ pub fn compress_openai_chat_request(if !strategies.contains(&strategy) {strategies.push(strategy);}+ if let Some(slot) = per_strategy_tokens+ .iter_mut()+ .find(|s| s.strategy == strategy)+ {+ slot.original_tokens += original_tokens;+ slot.compressed_tokens += compressed_tokens;+ } else {+ per_strategy_tokens.push(PerStrategyTokens {+ strategy,+ original_tokens,+ compressed_tokens,+ });+ }+ }+ BlockAction::RejectedNotSmaller { strategy, .. } => {+ crate::observability::record_compression_rejected_by_token_check(strategy);}BlockAction::CompressorError {strategy,@@ -246,6 +269,7 @@ pub fn compress_openai_chat_request(tokens_after: compressed_tokens_total,strategies_applied: strategies,markers_inserted: Vec::new(),+ per_strategy_tokens,}}Err(LiveZoneError::BodyNotJson(_)) => {diff --git a/crates/headroom-proxy/src/compression/live_zone_responses.rs b/crates/headroom-proxy/src/compression/live_zone_responses.rsindex ef4e58378..82380d86a 100644--- a/crates/headroom-proxy/src/compression/live_zone_responses.rs+++ b/crates/headroom-proxy/src/compression/live_zone_responses.rs@@ -43,7 +43,7 @@ use serde_json::Value;use crate::cache_stabilization::tool_def_normalize::{any_tool_has_cache_control, sort_schema_keys_recursive, sort_tools_deterministically,};-use crate::compression::{Outcome, PassthroughReason};+use crate::compression::{Outcome, PassthroughReason, PerStrategyTokens};use crate::config::CompressionMode;/// OpenAI Responses live-zone compression entry point.@@ -170,6 +170,7 @@ pub fn compress_openai_responses_request(tokens_after: 0,strategies_applied: normalization_applied.strategies(),markers_inserted: Vec::new(),+ per_strategy_tokens: Vec::new(),};}Outcome::NoCompression@@ -177,12 +178,15 @@ pub fn compress_openai_responses_request(Ok(LiveZoneOutcome::Modified { new_body, manifest }) => {// Aggregate per-block savings for the structured log.// Mirrors the Chat Completions sibling so dashboards- // don't need provider-specific shapes.+ // don't need provider-specific shapes. H1 + C5: per-+ // strategy token accumulation + rejected-token-check+ // counter.let mut original_bytes_total: usize = 0;let mut compressed_bytes_total: usize = 0;let mut original_tokens_total: usize = 0;let mut compressed_tokens_total: usize = 0;let mut strategies: Vec<&'static str> = Vec::new();+ let mut per_strategy_tokens: Vec<PerStrategyTokens> = Vec::new();let mut had_compressor_error = false;for entry in &manifest.block_outcomes {match entry.action {@@ -200,6 +204,22 @@ pub fn compress_openai_responses_request(if !strategies.contains(&strategy) {strategies.push(strategy);}+ if let Some(slot) = per_strategy_tokens+ .iter_mut()+ .find(|s| s.strategy == strategy)+ {+ slot.original_tokens += original_tokens;+ slot.compressed_tokens += compressed_tokens;+ } else {+ per_strategy_tokens.push(PerStrategyTokens {+ strategy,+ original_tokens,+ compressed_tokens,+ });+ }+ }+ BlockAction::RejectedNotSmaller { strategy, .. } => {+ crate::observability::record_compression_rejected_by_token_check(strategy);}BlockAction::CompressorError {strategy,@@ -257,6 +277,7 @@ pub fn compress_openai_responses_request(tokens_after: compressed_tokens_total,strategies_applied: strategies,markers_inserted: Vec::new(),+ per_strategy_tokens,}}Err(LiveZoneError::BodyNotJson(_)) => {diff --git a/crates/headroom-proxy/src/compression/mod.rs b/crates/headroom-proxy/src/compression/mod.rsindex 24bcbb512..af441e39a 100644--- a/crates/headroom-proxy/src/compression/mod.rs+++ b/crates/headroom-proxy/src/compression/mod.rs@@ -44,7 +44,9 @@ pub mod model_limits;// itself stays through B1 → B2 transition for parallel review;// `compress_anthropic_request` is sourced from the live-zone module.pub use anthropic::resolve_frozen_count;-pub use live_zone_anthropic::{compress_anthropic_request, Outcome, PassthroughReason};+pub use live_zone_anthropic::{+ compress_anthropic_request, Outcome, PassthroughReason, PerStrategyTokens,+};pub use live_zone_openai::{compress_openai_chat_request, should_skip_compression, SkipCompressionReason,};diff --git a/crates/headroom-proxy/src/handlers/responses.rs b/crates/headroom-proxy/src/handlers/responses.rsindex 9376c993f..da18e60c2 100644--- a/crates/headroom-proxy/src/handlers/responses.rs+++ b/crates/headroom-proxy/src/handlers/responses.rs@@ -120,12 +120,18 @@ pub async fn handle_responses(// bodies do NOT fabricate a tier — per realignment build-// constraint "no silent fallbacks", we just skip the emit and// log at debug.+ //+ // C1 fix: every raw value is validated against the bounded+ // `service_tier` vocabulary BEFORE being used as a label so a+ // malicious client cannot blow up label cardinality with+ // arbitrary strings.if let Some(tier) = extract_request_service_tier(&body) {let request_id_for_metric = headers.get("x-request-id").and_then(|v| v.to_str().ok()).unwrap_or("<no-request-id>");- observability::record_service_tier(&tier, request_id_for_metric);+ let bucketed = crate::observability::metric_names::service_tier::validate(&tier);+ observability::record_service_tier(bucketed, request_id_for_metric);} else {tracing::debug!(event = "service_tier_skipped",@@ -167,10 +173,12 @@ pub async fn handle_responses(/// Phase G PR-G3: best-effort parse of `service_tier` from the/// inbound request body. Returns `None` when the body is not valid/// JSON, not an object, or lacks the field. The spec defines the-/// field as a string ∈ {auto, default, flex, on_demand, priority};-/// we do NOT validate against that set here so wire-format drift-/// (e.g. OpenAI adding a tier we haven't enumerated yet) surfaces-/// in the metric rather than getting silently dropped.+/// field as a string ∈ {auto, default, flex, on_demand, priority,+/// scale}; the returned raw string is normalised against the+/// bounded vocabulary at the call site via+/// [`crate::observability::metric_names::service_tier::validate`]+/// so an arbitrary inbound value cannot drive metric-label+/// cardinality unbounded (C1 fix).fn extract_request_service_tier(body: &Bytes) -> Option<String> {let v: serde_json::Value = serde_json::from_slice(body).ok()?;v.get("service_tier")diff --git a/crates/headroom-proxy/src/observability/cache_hit_rate.rs b/crates/headroom-proxy/src/observability/cache_hit_rate.rsindex ef152be29..4e72e911a 100644--- a/crates/headroom-proxy/src/observability/cache_hit_rate.rs+++ b/crates/headroom-proxy/src/observability/cache_hit_rate.rs@@ -118,6 +118,31 @@ pub fn compute_hit_rate(Some(cache_read_input_tokens as f64 / denom as f64)}+/// H2 gate: should we observe a cache-hit-rate sample for this+/// Anthropic session?+///+/// Returns `Some(rate)` ONLY when the stream completed cleanly+/// (`state.status == MessageStop`) AND the denominator is non-zero.+/// A client disconnect mid-stream closes the channel too — without+/// this gate we'd observe a half-finished session that has only+/// `message_start` usage and pollute the histogram with garbage.+///+/// Extracted from `proxy.rs::run_sse_state_machine` so the H2+/// contract is unit-testable independent of the global Prometheus+/// registry (which parallel tests share).+pub fn compute_anthropic_session_hit_rate(+ state: &crate::sse::anthropic::AnthropicStreamState,+) -> Option<f64> {+ if state.status != crate::sse::anthropic::StreamStatus::MessageStop {+ return None;+ }+ compute_hit_rate(+ state.usage.input_tokens,+ state.usage.cache_read_input_tokens,+ state.usage.cache_creation_input_tokens,+ )+}+/// Observe one per-session sample.////// `provider` MUST be one of the [`provider`] constants — callers@@ -125,11 +150,31 @@ pub fn compute_hit_rate(/// do not validate the label here because the cardinality is bounded/// by the static call sites; an invalid label would be a bug, not a/// runtime mismatch.+///+/// M3 fix: NaN / non-finite inputs are loud-skipped instead of+/// silently observed. `f64::clamp(0.0, 1.0)` returns NaN when the+/// input is NaN, so the prior implementation could pollute the+/// histogram with NaN samples in release builds (the debug_assert+/// was compiled out). Per "no silent fallbacks", an unexpected NaN+/// surfaces in the logs rather than getting eaten.pub fn observe(provider: &'static str, request_id: &str, hit_rate: f64) {+ if !hit_rate.is_finite() {+ tracing::warn!(+ event = "cache_hit_rate_non_finite",+ metric = METRIC_PROXY_CACHE_HIT_RATE_PER_SESSION,+ provider = provider,+ request_id = %request_id,+ hit_rate = hit_rate,+ "refusing to observe a non-finite cache hit rate; this is a caller bug"+ );+ return;+ }debug_assert!((0.0..=1.0).contains(&hit_rate),"cache hit rate must be in [0.0, 1.0]; got {hit_rate}");+ // After the is_finite guard, clamp can only normalise legitimate+ // edge-of-range floats (1.0 + epsilon etc.) and never produces NaN.let clamped = hit_rate.clamp(0.0, 1.0);histogram(super::prometheus::registry()).with_label_values(&[provider])@@ -175,4 +220,121 @@ mod tests {// observation, not coerce to 0.0.assert!(compute_hit_rate(0, 0, 0).is_none());}++ #[test]+ fn observe_nan_skipped_loudly() {+ // M3: a NaN input must NOT reach the histogram. The+ // pre-fix code clamped via `f64::clamp(0.0, 1.0)` which+ // returns NaN for NaN input — a NaN sample in release+ // builds was the bug. After the fix we log + skip.+ // The histogram count for this synthetic provider stays+ // at whatever it was before the call.+ let label = "test_nan_provider_v1";+ // Drive a guaranteed-clean session count via a real+ // observation, then push a NaN and assert no count move.+ observe(label, "req-nan-baseline", 0.5);+ let before = histogram(super::super::prometheus::registry())+ .with_label_values(&[label])+ .get_sample_count();+ observe(label, "req-nan-attempt", f64::NAN);+ let after = histogram(super::super::prometheus::registry())+ .with_label_values(&[label])+ .get_sample_count();+ assert_eq!(+ before, after,+ "NaN must not be observed; expected count unchanged from {before} to {after}"+ );+ }++ #[test]+ fn h2_aborted_anthropic_stream_returns_none() {+ // H2: a stream that closes without `message_stop` (Open+ // state) MUST NOT produce a sample, regardless of usage+ // values.+ use crate::sse::anthropic::{AnthropicStreamState, StreamStatus, UsageBuilder};+ let state = AnthropicStreamState {+ status: StreamStatus::Open,+ usage: UsageBuilder {+ input_tokens: 800,+ cache_read_input_tokens: 200,+ output_tokens: 50,+ cache_creation_input_tokens: 0,+ },+ ..Default::default()+ };+ assert!(+ compute_anthropic_session_hit_rate(&state).is_none(),+ "H2 gate must skip emission for non-completed stream"+ );+ }++ #[test]+ fn h2_errored_anthropic_stream_returns_none() {+ // H2: an Errored stream is also not "completed cleanly" —+ // skip the observation.+ use crate::sse::anthropic::{AnthropicStreamState, StreamStatus, UsageBuilder};+ let state = AnthropicStreamState {+ status: StreamStatus::Errored,+ usage: UsageBuilder {+ input_tokens: 800,+ cache_read_input_tokens: 200,+ output_tokens: 50,+ cache_creation_input_tokens: 0,+ },+ ..Default::default()+ };+ assert!(+ compute_anthropic_session_hit_rate(&state).is_none(),+ "H2 gate must skip emission for errored stream"+ );+ }++ #[test]+ fn h2_completed_anthropic_stream_returns_rate() {+ // H2 positive case: MessageStop + non-zero usage → return+ // the rate so the caller can observe it.+ use crate::sse::anthropic::{AnthropicStreamState, StreamStatus, UsageBuilder};+ let state = AnthropicStreamState {+ status: StreamStatus::MessageStop,+ usage: UsageBuilder {+ input_tokens: 800,+ cache_read_input_tokens: 200,+ output_tokens: 50,+ cache_creation_input_tokens: 0,+ },+ ..Default::default()+ };+ let rate = compute_anthropic_session_hit_rate(&state).expect("completed stream emits");+ // 200 / (800 + 200 + 0) = 0.2+ assert!((rate - 0.2).abs() < 1e-9);+ }++ #[test]+ fn h2_completed_but_zero_denominator_returns_none() {+ // Even on a completed stream, a zero-token request returns+ // None — per "no silent fallbacks", no synthesised 0.0.+ use crate::sse::anthropic::{AnthropicStreamState, StreamStatus, UsageBuilder};+ let state = AnthropicStreamState {+ status: StreamStatus::MessageStop,+ usage: UsageBuilder::default(),+ ..Default::default()+ };+ assert!(compute_anthropic_session_hit_rate(&state).is_none());+ }++ #[test]+ fn observe_infinity_skipped_loudly() {+ // Same contract for +/- infinity.+ let label = "test_inf_provider_v1";+ observe(label, "req-inf-baseline", 0.5);+ let before = histogram(super::super::prometheus::registry())+ .with_label_values(&[label])+ .get_sample_count();+ observe(label, "req-pos-inf", f64::INFINITY);+ observe(label, "req-neg-inf", f64::NEG_INFINITY);+ let after = histogram(super::super::prometheus::registry())+ .with_label_values(&[label])+ .get_sample_count();+ assert_eq!(before, after);+ }}diff --git a/crates/headroom-proxy/src/observability/metric_names.rs b/crates/headroom-proxy/src/observability/metric_names.rsindex 4edefb18b..7708f92f5 100644--- a/crates/headroom-proxy/src/observability/metric_names.rs+++ b/crates/headroom-proxy/src/observability/metric_names.rs@@ -94,29 +94,15 @@ pub const METRIC_PROXY_RESPONSE_STATUS_COUNT_TOTAL_HELP: &str ='incomplete' detail lands in the structured log paired with each \increment.";-// ---------- proxy_image_generation_call_log_redacted_total ------------pub const METRIC_PROXY_IMAGE_GENERATION_CALL_LOG_REDACTED_TOTAL: &str =- "proxy_image_generation_call_log_redacted_total";-pub const METRIC_PROXY_IMAGE_GENERATION_CALL_LOG_REDACTED_TOTAL_HELP: &str =- "Count of base64-encoded image payloads redacted from request logs. \- Driven from the Python proxy's request logger (large multi-MB \- base64 strings replaced with size-only placeholders).";--// ---------- wrap_rtk_invocations_total ------------pub const METRIC_WRAP_RTK_INVOCATIONS_TOTAL: &str = "wrap_rtk_invocations_total";-pub const METRIC_WRAP_RTK_INVOCATIONS_TOTAL_HELP: &str =- "Count of RTK invocations observed via the wrap-CLI polling of \- `rtk gain --format json`, broken down by the rewritten command \- (`git`, `ls`, `cargo`, …).";--// ---------- wrap_rtk_tokens_saved_per_session ------------pub const METRIC_WRAP_RTK_TOKENS_SAVED_PER_SESSION: &str = "wrap_rtk_tokens_saved_per_session";-pub const METRIC_WRAP_RTK_TOKENS_SAVED_PER_SESSION_HELP: &str =- "Tokens saved by RTK during one wrap session, observed once at \- session end. Histogram so dashboards can render a distribution.";+// Phase G PR-G3 remediation (C3 + C4): the metric-name constants+// for `proxy_image_generation_call_log_redacted_total`,+// `wrap_rtk_invocations_total`, and `wrap_rtk_tokens_saved_per_session`+// were removed because the underlying counters had no production+// emit site on the Rust side. The same metrics are exported by the+// Python proxy (`headroom/proxy/prometheus_metrics.py`) which is the+// natural owner: image redaction is a Python-proxy operation and RTK+// invocation tracking lives in the wrap CLI, both Python-side+// surfaces. See `docs/observability.md`.// ---------- shared label keys ----------@@ -126,20 +112,54 @@ pub const LABEL_CONTENT_TYPE: &str = "content_type";pub const LABEL_PATH: &str = "path";pub const LABEL_TIER: &str = "tier";pub const LABEL_STATUS: &str = "status";-pub const LABEL_TOOL: &str = "tool";// ---------- bounded label vocabularies ----------/// OpenAI service-tier values per the Responses API spec-/// (`service_tier` field on the response object). The proxy logs-/// anything outside this set under the literal value so wire-format-/// drift is loud rather than silently bucketed.+/// (`service_tier` field on the response object). The metric label+/// vocabulary is **strictly** this set plus a `"scale"` value+/// (documented in OpenAI's tier-pricing page) and a sentinel+/// `"other"` bucket for anything else, so a malicious client posting+/// `{"service_tier":"<random>"}` per request cannot blow up+/// cardinality.pub mod service_tier {pub const AUTO: &str = "auto";pub const DEFAULT: &str = "default";pub const FLEX: &str = "flex";pub const ON_DEMAND: &str = "on_demand";pub const PRIORITY: &str = "priority";+ pub const SCALE: &str = "scale";+ /// Sentinel for any unknown / unrecognised tier value. Prevents+ /// label-cardinality DoS from arbitrary inbound JSON.+ pub const OTHER: &str = "other";++ /// Validate an inbound `service_tier` string against the bounded+ /// vocabulary. Returns the matching `&'static` constant or+ /// [`OTHER`] for any unrecognised value (with a tracing::warn so+ /// wire-format drift is loud rather than silently bucketed).+ ///+ /// The matching is case-sensitive — the OpenAI spec is+ /// case-sensitive on these strings; a case-different value is+ /// treated as drift, not as the same tier.+ pub fn validate(raw: &str) -> &'static str {+ match raw {+ AUTO => AUTO,+ DEFAULT => DEFAULT,+ FLEX => FLEX,+ ON_DEMAND => ON_DEMAND,+ PRIORITY => PRIORITY,+ SCALE => SCALE,+ _ => {+ tracing::warn!(+ event = "service_tier_unknown",+ raw = %raw,+ bucket = OTHER,+ "unknown service_tier value bucketed to 'other' to bound cardinality"+ );+ OTHER+ }+ }+ }}/// OpenAI Responses terminal-status vocabulary. `in_progress` is thediff --git a/crates/headroom-proxy/src/observability/mod.rs b/crates/headroom-proxy/src/observability/mod.rsindex 295a52b9f..d8ad2c7a7 100644--- a/crates/headroom-proxy/src/observability/mod.rs+++ b/crates/headroom-proxy/src/observability/mod.rs@@ -63,7 +63,6 @@ pub use compression_ratio::{record_rejected_by_token_check as record_compression_rejected_by_token_check,};pub use proxy_metrics::{- extract_rate_limit_snapshot, record_image_redacted, record_passthrough_bytes_modified,- record_rate_limit_snapshot, record_response_status, record_rtk_invocation, record_service_tier,- RateLimitSnapshot,+ extract_rate_limit_snapshot, record_passthrough_bytes_modified, record_rate_limit_snapshot,+ record_response_status, record_service_tier, RateLimitSnapshot,};diff --git a/crates/headroom-proxy/src/observability/prometheus.rs b/crates/headroom-proxy/src/observability/prometheus.rsindex ec23a7605..bd6307d85 100644--- a/crates/headroom-proxy/src/observability/prometheus.rs+++ b/crates/headroom-proxy/src/observability/prometheus.rs@@ -217,21 +217,50 @@ pub async fn handle_metrics() -> Response {// Phase G PR-G3: same idea for the new proxy-wide metric families.// Lazy `OnceLock`-backed singletons; touching each here forces- // registration on first scrape so HELP/TYPE lines appear even- // before traffic has driven a single increment.+ // registration on first scrape.+ //+ // H3 fix: registration alone is NOT enough — the prometheus+ // crate's v0.13 `gather()` skips empty MetricVec families+ // entirely (no HELP/TYPE lines either). Operators who curl+ // /metrics on a fresh boot would otherwise see NO trace of the+ // catalogue. We force-touch each counter / gauge MetricVec+ // with a sentinel `__init__` label so HELP/TYPE + a zero row+ // appear from boot and dashboards/alarms have a predictable+ // scrape shape. Counters with the `__init__` label increment+ // by 0, so the alarm-able "must stay 0" semantic of+ // `proxy_passthrough_bytes_modified_total` is preserved+ // (the family becomes visible, the rate stays 0).+ //+ // Histograms are NOT force-zeroed: a synthetic `observe(0.0)`+ // would pollute the per-label distribution. The two histogram+ // families (`proxy_cache_hit_rate_per_session` and+ // `proxy_compression_ratio_by_strategy`) only surface in the+ // scrape after the first real session, by design.let reg = registry();let _ = super::cache_hit_rate::histogram(reg);let _ = super::compression_ratio::ratio_histogram(reg);- let _ = super::compression_ratio::rejected_counter(reg);- let _ = super::proxy_metrics::passthrough_bytes_modified_counter(reg);- let _ = super::proxy_metrics::rate_limit_remaining_requests_gauge(reg);- let _ = super::proxy_metrics::rate_limit_remaining_tokens_gauge(reg);- let _ = super::proxy_metrics::rate_limit_remaining_input_tokens_gauge(reg);- let _ = super::proxy_metrics::rate_limit_remaining_output_tokens_gauge(reg);- let _ = super::proxy_metrics::service_tier_counter(reg);- let _ = super::proxy_metrics::response_status_counter(reg);- let _ = super::proxy_metrics::image_redacted_counter(reg);- let _ = super::proxy_metrics::rtk_invocations_counter(reg);+ let rejected_counter = super::compression_ratio::rejected_counter(reg);+ let passthrough_counter = super::proxy_metrics::passthrough_bytes_modified_counter(reg);+ let rl_requests_gauge = super::proxy_metrics::rate_limit_remaining_requests_gauge(reg);+ let rl_tokens_gauge = super::proxy_metrics::rate_limit_remaining_tokens_gauge(reg);+ let rl_input_gauge = super::proxy_metrics::rate_limit_remaining_input_tokens_gauge(reg);+ let rl_output_gauge = super::proxy_metrics::rate_limit_remaining_output_tokens_gauge(reg);+ let tier_counter = super::proxy_metrics::service_tier_counter(reg);+ let status_counter = super::proxy_metrics::response_status_counter(reg);++ const INIT_SENTINEL: &str = "__init__";+ rejected_counter+ .with_label_values(&[INIT_SENTINEL])+ .inc_by(0);+ passthrough_counter+ .with_label_values(&[INIT_SENTINEL])+ .inc_by(0);+ rl_requests_gauge.with_label_values(&[INIT_SENTINEL]).set(0);+ rl_tokens_gauge.with_label_values(&[INIT_SENTINEL]).set(0);+ rl_input_gauge.with_label_values(&[INIT_SENTINEL]).set(0);+ rl_output_gauge.with_label_values(&[INIT_SENTINEL]).set(0);+ tier_counter.with_label_values(&[INIT_SENTINEL]).inc_by(0);+ status_counter.with_label_values(&[INIT_SENTINEL]).inc_by(0);let metric_families = registry().gather();let mut buffer = Vec::with_capacity(2048);diff --git a/crates/headroom-proxy/src/observability/proxy_metrics.rs b/crates/headroom-proxy/src/observability/proxy_metrics.rsindex 527e64ec6..371b2887f 100644--- a/crates/headroom-proxy/src/observability/proxy_metrics.rs+++ b/crates/headroom-proxy/src/observability/proxy_metrics.rs@@ -13,9 +13,7 @@ use std::sync::OnceLock;use prometheus::{IntCounterVec, IntGaugeVec, Opts, Registry};use super::metric_names::{- LABEL_PATH, LABEL_PROVIDER, LABEL_STATUS, LABEL_TIER, LABEL_TOOL,- METRIC_PROXY_IMAGE_GENERATION_CALL_LOG_REDACTED_TOTAL,- METRIC_PROXY_IMAGE_GENERATION_CALL_LOG_REDACTED_TOTAL_HELP,+ LABEL_PATH, LABEL_PROVIDER, LABEL_STATUS, LABEL_TIER,METRIC_PROXY_PASSTHROUGH_BYTES_MODIFIED_TOTAL,METRIC_PROXY_PASSTHROUGH_BYTES_MODIFIED_TOTAL_HELP,METRIC_PROXY_RATE_LIMIT_REMAINING_INPUT_TOKENS,@@ -26,7 +24,6 @@ use super::metric_names::{METRIC_PROXY_RATE_LIMIT_REMAINING_TOKENS, METRIC_PROXY_RATE_LIMIT_REMAINING_TOKENS_HELP,METRIC_PROXY_RESPONSE_STATUS_COUNT_TOTAL, METRIC_PROXY_RESPONSE_STATUS_COUNT_TOTAL_HELP,METRIC_PROXY_SERVICE_TIER_COUNT_TOTAL, METRIC_PROXY_SERVICE_TIER_COUNT_TOTAL_HELP,- METRIC_WRAP_RTK_INVOCATIONS_TOTAL, METRIC_WRAP_RTK_INVOCATIONS_TOTAL_HELP,};// ---------- proxy_passthrough_bytes_modified_total{path} ----------@@ -269,7 +266,11 @@ pub fn record_response_status(status: &str, reason: Option<&str>, request_id: &sresponse_status_counter(super::prometheus::registry()).with_label_values(&[status]).inc();- tracing::info!(+ // Optional-3: aligned with the peer `record_*` helpers in this+ // module which all use `debug!` for the metric-correlation log+ // line. INFO was inconsistent and produced extra log volume+ // during normal Responses traffic.+ tracing::debug!(event = "metric_recorded",metric = METRIC_PROXY_RESPONSE_STATUS_COUNT_TOTAL,status = %status,@@ -279,74 +280,18 @@ pub fn record_response_status(status: &str, reason: Option<&str>, request_id: &s);}-// ---------- proxy_image_generation_call_log_redacted_total ------------pub fn image_redacted_counter(registry: &Registry) -> &'static IntCounterVec {- static COUNTER: OnceLock<IntCounterVec> = OnceLock::new();- COUNTER.get_or_init(|| {- let opts = Opts::new(- METRIC_PROXY_IMAGE_GENERATION_CALL_LOG_REDACTED_TOTAL,- METRIC_PROXY_IMAGE_GENERATION_CALL_LOG_REDACTED_TOTAL_HELP,- );- // Unlabelled metric — but `IntCounterVec` with zero labels- // is still the right type for symmetry with the rest of the- // module. A bare `IntCounter` would diverge in registration- // shape and read site.- let counter = IntCounterVec::new(opts, &[])- .expect("proxy_image_generation_call_log_redacted_total descriptor is well-formed");- registry- .register(Box::new(counter.clone()))- .expect("proxy_image_generation_call_log_redacted_total registers exactly once");- counter- })-}--pub fn record_image_redacted() {- image_redacted_counter(super::prometheus::registry())- .with_label_values(&[])- .inc();- tracing::debug!(- event = "metric_recorded",- metric = METRIC_PROXY_IMAGE_GENERATION_CALL_LOG_REDACTED_TOTAL,- "incremented proxy_image_generation_call_log_redacted_total"- );-}--// ---------- wrap_rtk_invocations_total{tool} -----------//-// RTK lives on the wrap-CLI side; the proxy never invokes it. The-// counter is registered here so the central /metrics scrape exposes-// it and the wrap-side tail can `inc()` via the registry. The wrap-// side polls `rtk gain --format json` and increments by the delta.--pub fn rtk_invocations_counter(registry: &Registry) -> &'static IntCounterVec {- static COUNTER: OnceLock<IntCounterVec> = OnceLock::new();- COUNTER.get_or_init(|| {- let opts = Opts::new(- METRIC_WRAP_RTK_INVOCATIONS_TOTAL,- METRIC_WRAP_RTK_INVOCATIONS_TOTAL_HELP,- );- let counter = IntCounterVec::new(opts, &[LABEL_TOOL])- .expect("wrap_rtk_invocations_total descriptor is well-formed");- registry- .register(Box::new(counter.clone()))- .expect("wrap_rtk_invocations_total registers exactly once");- counter- })-}--pub fn record_rtk_invocation(tool: &str, delta: u64) {- rtk_invocations_counter(super::prometheus::registry())- .with_label_values(&[tool])- .inc_by(delta);- tracing::debug!(- event = "metric_recorded",- metric = METRIC_WRAP_RTK_INVOCATIONS_TOTAL,- tool = %tool,- delta = delta,- "incremented wrap_rtk_invocations_total"- );-}+// Phase G PR-G3 remediation (C3 + C4): the image-redacted counter+// and the wrap_rtk_invocations counter were originally registered+// here but neither had a production emit site that crossed the+// Python/Rust boundary. Both have moved Python-side+// (`headroom.proxy.request_logger::redactions_total` and+// `headroom.cli.wrap_rtk_metrics::rtk_invocation_counts`) and the+// Python proxy's `/metrics` exporter surfaces them — see+// `docs/observability.md` for the placement decision. Keeping a+// dead Rust counter would (a) violate the "no dead metrics+// registered" review finding and (b) mislead Phase H canary+// dashboards into expecting two scrape sources for what is really+// one Python-side counter.#[cfg(test)]mod tests {diff --git a/crates/headroom-proxy/src/proxy.rs b/crates/headroom-proxy/src/proxy.rsindex e81294442..d3fe821cc 100644--- a/crates/headroom-proxy/src/proxy.rs+++ b/crates/headroom-proxy/src/proxy.rs@@ -730,22 +730,24 @@ pub(crate) async fn forward_http(}};+ // C2 fix: snapshot the original buffered byte-length AND the+ // dispatcher's "is this a passthrough arm?" decision BEFORE+ // `outcome` is consumed by the match below. The+ // passthrough-bytes-modified alarm fires when a path that+ // promised byte-equal passthrough produces a different+ // length downstream.+ let original_buffered_len = buffered.len();+ let outcome_is_passthrough_class = matches!(+ outcome,+ compression::Outcome::NoCompression | compression::Outcome::Passthrough { .. }+ );let body_to_send = match outcome {compression::Outcome::NoCompression => {// PR-B2: forward the *original* buffered bytes. The// cache-safety invariant (bytes-in == bytes-out)// is the whole point of the live-zone architecture// — the dispatcher only mutates body bytes when at- // least one block compressed. PR-B2's no-op- // skeleton always lands here. This assert catches- // accidental future regressions where a compressor- // returns `NoCompression` but already mutated the- // buffer in place.- debug_assert_eq!(- buffered.len(),- buffered.len(),- "buffered bytes length must remain stable on the NoCompression path"- );+ // least one block compressed.buffered}// PR-B3+ produces `Compressed` from the live-zone@@ -758,6 +760,7 @@ pub(crate) async fn forward_http(tokens_after,strategies_applied,markers_inserted,+ per_strategy_tokens,} => {tracing::info!(request_id = %request_id,@@ -769,25 +772,50 @@ pub(crate) async fn forward_http(markers = markers_inserted.len(),"compression applied");- // Phase G PR-G3: emit one+ // Phase G PR-G3 + H1: emit one// `proxy_compression_ratio_by_strategy` sample per- // strategy actually applied. We do NOT have a- // per-block content_type breakdown here — the- // aggregate `tokens_before` / `tokens_after` is- // strategy-wise rather than content-type-wise, so we- // label `content_type=aggregate`. Per-block fanout- // would require threading the manifest through to- // this layer; that's a future PR if dashboards need- // it.- if tokens_before > 0 && tokens_after < tokens_before {- for strategy in &strategies_applied {+ // strategy with the *strategy's own* before/after+ // token counts. The pre-H1 code emitted the same+ // aggregate ratio for every strategy in+ // `strategies_applied`, so Phase H per-strategy+ // dashboards read garbage when multiple strategies+ // ran on one body. We now plumb per-strategy tokens+ // from the manifest at the wrapper site+ // (`live_zone_anthropic`, `live_zone_openai`,+ // `live_zone_responses`).+ //+ // Fallback: when `per_strategy_tokens` is empty —+ // i.e. the Outcome came from a Phase E+ // normalization pass that doesn't track per-strategy+ // tokens — we emit one aggregate-labelled sample so+ // dashboards still see *that* a compression ran. We+ // log loudly so this is visible.+ if !per_strategy_tokens.is_empty() {+ for entry in &per_strategy_tokens {crate::observability::observe_compression_ratio(- strategy,+ entry.strategy,"aggregate",- tokens_before,- tokens_after,+ entry.original_tokens,+ entry.compressed_tokens,);}+ } else if tokens_before > 0 && tokens_after < tokens_before {+ tracing::debug!(+ event = "compression_ratio_emit_aggregate_only",+ request_id = %request_id,+ path = %path_for_log,+ strategies = ?strategies_applied,+ reason = "no_per_strategy_tokens",+ "emitting one aggregate-labelled compression_ratio sample because \+ the dispatcher did not surface per-strategy token counts \+ (Phase E normalization paths)"+ );+ crate::observability::observe_compression_ratio(+ "aggregate",+ "aggregate",+ tokens_before,+ tokens_after,+ );}body}@@ -802,6 +830,24 @@ pub(crate) async fn forward_http(}};+ // C2 fix: cache-safety alarm. When the dispatcher returned+ // `NoCompression` or `Passthrough`, the post-dispatcher body+ // MUST be byte-length-equal to the original buffered body.+ // Any delta is an accidental cache-poisoning regression and+ // the alarm metric `proxy_passthrough_bytes_modified_total{path}`+ // fires with the byte delta as its increment. We check BEFORE+ // the PR-E4 prompt_cache_key injector runs because that+ // injector is a legitimate, intentional byte mutation gated+ // on PAYG; it must not trip the alarm.+ if outcome_is_passthrough_class && body_to_send.len() != original_buffered_len {+ let delta = body_to_send.len().abs_diff(original_buffered_len) as u64;+ crate::observability::record_passthrough_bytes_modified(+ &path_for_log,+ delta,+ &request_id,+ );+ }+// PR-E4: OpenAI `prompt_cache_key` auto-injection.//// Universal safety contract: only mutate when the caller@@ -1242,17 +1288,12 @@ async fn run_sse_state_machine(}}}- // Phase G PR-G3: emit per-session cache-hit-rate from- // the final accumulated `usage`. Anthropic's- // `message_delta` strictly grows token counts, so the- // accumulator's final values are the right denominator.- // `compute_cache_hit_rate` returns `None` for zero- // denominators → log + skip, never synthesise a sample.- match crate::observability::compute_cache_hit_rate(- state.usage.input_tokens,- state.usage.cache_read_input_tokens,- state.usage.cache_creation_input_tokens,- ) {+ // Phase G PR-G3 + H2: emit per-session cache-hit-rate+ // ONLY when the stream completed cleanly with+ // `message_stop`. The gate is encapsulated by the+ // pure function `compute_anthropic_session_hit_rate`+ // so the H2 contract has a unit-testable surface.+ match crate::observability::cache_hit_rate::compute_anthropic_session_hit_rate(&state) {Some(rate) => {crate::observability::observe_cache_hit_rate(crate::observability::cache_hit_rate_provider::ANTHROPIC,@@ -1265,8 +1306,11 @@ async fn run_sse_state_machine(event = "cache_hit_rate_skipped",request_id = %request_id,provider = "anthropic",- reason = "zero_denominator",- "skipping proxy_cache_hit_rate_per_session: no input tokens"+ status = ?state.status,+ input_tokens = state.usage.input_tokens,+ cache_read_input_tokens = state.usage.cache_read_input_tokens,+ cache_creation_input_tokens = state.usage.cache_creation_input_tokens,+ "skipping proxy_cache_hit_rate_per_session: H2 gate or zero denominator");}}@@ -1311,7 +1355,9 @@ async fn run_sse_state_machine(// chunk. OpenAI only emits this when// `stream_options.include_usage = true`; absence is a// signal, not a fallback condition — `usage = None` →- // skip.+ // skip. The H2 gate is implicit here: the final usage+ // chunk only arrives when the stream completed (it's+ // OpenAI's terminal-status equivalent).if let Some(usage) = &state.usage {let input_tokens = usage.get("prompt_tokens")@@ -1322,29 +1368,48 @@ async fn run_sse_state_machine(.and_then(|d| d.get("cached_tokens")).and_then(|v| v.as_u64()).unwrap_or(0);- // OpenAI's `prompt_tokens` already INCLUDES cached- // tokens (per Chat Completions API docs), so the- // denominator is `prompt_tokens`, not the sum. The- // numerator is `cached_tokens`; `input_tokens` arg to- // `compute_cache_hit_rate` carries the *non-cached*- // portion (denom-only), so we synthesise that here.- let non_cached = input_tokens.saturating_sub(cached_tokens);- match crate::observability::compute_cache_hit_rate(non_cached, cached_tokens, 0) {- Some(rate) => {- crate::observability::observe_cache_hit_rate(- crate::observability::cache_hit_rate_provider::OPENAI_CHAT,- &request_id,- rate,- );- }- None => {- tracing::debug!(- event = "cache_hit_rate_skipped",- request_id = %request_id,- provider = "openai_chat",- reason = "zero_denominator",- "skipping proxy_cache_hit_rate_per_session: no input tokens"- );+ // M1: `cached_tokens > input_tokens` is a wire-+ // format pathology — log + skip instead of silently+ // clamping (saturating_sub would yield 0 → fake 1.0+ // hit-rate sample).+ if cached_tokens > input_tokens {+ tracing::warn!(+ event = "cache_hit_rate_skipped",+ request_id = %request_id,+ provider = "openai_chat",+ reason = "cached_gt_input",+ input_tokens = input_tokens,+ cached_tokens = cached_tokens,+ "skipping proxy_cache_hit_rate_per_session: cached_tokens > prompt_tokens \+ (wire-format pathology; clamping would synthesise a bad sample)"+ );+ } else {+ // OpenAI's `prompt_tokens` already INCLUDES cached+ // tokens (per Chat Completions API docs), so the+ // denominator is `prompt_tokens`, not the sum. The+ // numerator is `cached_tokens`; `input_tokens` arg+ // to `compute_cache_hit_rate` carries the+ // *non-cached* portion (denom-only), so we+ // synthesise that here.+ let non_cached = input_tokens - cached_tokens;+ match crate::observability::compute_cache_hit_rate(non_cached, cached_tokens, 0)+ {+ Some(rate) => {+ crate::observability::observe_cache_hit_rate(+ crate::observability::cache_hit_rate_provider::OPENAI_CHAT,+ &request_id,+ rate,+ );+ }+ None => {+ tracing::debug!(+ event = "cache_hit_rate_skipped",+ request_id = %request_id,+ provider = "openai_chat",+ reason = "zero_denominator",+ "skipping proxy_cache_hit_rate_per_session: no input tokens"+ );+ }}}} else {@@ -1389,48 +1454,93 @@ async fn run_sse_state_machine(}}}- // Phase G PR-G3: cache hit rate + service_tier +- // response status from the final `response.completed`- // payload. The Responses API uses `input_tokens` /+ // Phase G PR-G3 + H2: cache hit rate + service_tier ++ // response status emit ONLY when the stream reached a+ // terminal status (`response.completed/failed/incomplete`).+ // Mid-stream client disconnects close the channel without+ // a terminal — `terminal_status().is_none()` then guards+ // emit so we don't observe garbage samples.+ //+ // The Responses API uses `input_tokens` /// `cached_input_tokens` shape (Responses-specific —// distinct from Chat Completions' `prompt_tokens`).- if let Some(usage) = &state.usage {- let input_tokens = usage- .get("input_tokens")- .and_then(|v| v.as_u64())- .unwrap_or(0);- let cached_tokens = usage- .get("input_tokens_details")- .and_then(|d| d.get("cached_tokens"))- .and_then(|v| v.as_u64())- .unwrap_or(0);- // Like Chat, `input_tokens` already INCLUDES cached- // tokens, so split for the helper.- let non_cached = input_tokens.saturating_sub(cached_tokens);- match crate::observability::compute_cache_hit_rate(non_cached, cached_tokens, 0) {- Some(rate) => {- crate::observability::observe_cache_hit_rate(- crate::observability::cache_hit_rate_provider::OPENAI_RESPONSES,- &request_id,- rate,- );- }- None => {- tracing::debug!(+ let stream_completed = state.terminal_status().is_some();+ if stream_completed {+ if let Some(usage) = &state.usage {+ let input_tokens = usage+ .get("input_tokens")+ .and_then(|v| v.as_u64())+ .unwrap_or(0);+ let cached_tokens = usage+ .get("input_tokens_details")+ .and_then(|d| d.get("cached_tokens"))+ .and_then(|v| v.as_u64())+ .unwrap_or(0);+ // M1: a cached count greater than input is a+ // wire-format pathology — usage shouldn't have+ // `cached > input` for OpenAI Responses. Per+ // "no silent fallbacks", log + skip the emit+ // instead of silently clamping.+ if cached_tokens > input_tokens {+ tracing::warn!(event = "cache_hit_rate_skipped",request_id = %request_id,provider = "openai_responses",- reason = "zero_denominator",- "skipping proxy_cache_hit_rate_per_session: no input tokens"+ reason = "cached_gt_input",+ input_tokens = input_tokens,+ cached_tokens = cached_tokens,+ "skipping proxy_cache_hit_rate_per_session: cached_tokens > input_tokens \+ (wire-format pathology; clamping would synthesise a bad sample)");+ } else {+ // Like Chat, `input_tokens` already INCLUDES cached+ // tokens, so split for the helper.+ let non_cached = input_tokens - cached_tokens;+ match crate::observability::compute_cache_hit_rate(+ non_cached,+ cached_tokens,+ 0,+ ) {+ Some(rate) => {+ crate::observability::observe_cache_hit_rate(+ crate::observability::cache_hit_rate_provider::OPENAI_RESPONSES,+ &request_id,+ rate,+ );+ }+ None => {+ tracing::debug!(+ event = "cache_hit_rate_skipped",+ request_id = %request_id,+ provider = "openai_responses",+ reason = "zero_denominator",+ "skipping proxy_cache_hit_rate_per_session: no input tokens"+ );+ }+ }}}+ } else {+ tracing::debug!(+ event = "cache_hit_rate_skipped",+ request_id = %request_id,+ provider = "openai_responses",+ reason = "stream_did_not_complete",+ "skipping proxy_cache_hit_rate_per_session: no terminal status seen"+ );}// Service tier + status are sourced from// `state.last_response_envelope` populated by the// ResponseState on `response.completed/failed/incomplete`.+ //+ // C1 fix: the tier value comes from the upstream response+ // body; even though the upstream is more trustworthy than+ // a client-side header, an unrecognised value would still+ // grow the metric vector unboundedly. We bucket through+ // the same validator the request-side handler uses.if let Some(tier) = state.service_tier.as_deref() {- crate::observability::record_service_tier(tier, &request_id);+ let bucketed = crate::observability::metric_names::service_tier::validate(tier);+ crate::observability::record_service_tier(bucketed, &request_id);}if let Some(status) = state.terminal_status() {crate::observability::record_response_status(diff --git a/crates/headroom-proxy/src/vertex/raw_predict.rs b/crates/headroom-proxy/src/vertex/raw_predict.rsindex 68a86472a..5059b2a44 100644--- a/crates/headroom-proxy/src/vertex/raw_predict.rs+++ b/crates/headroom-proxy/src/vertex/raw_predict.rs@@ -164,6 +164,7 @@ pub(crate) async fn forward_vertex_request(tokens_after,strategies_applied,markers_inserted,+ ..} => {tracing::info!(event = "vertex_compression_applied",diff --git a/crates/headroom-proxy/tests/integration_metrics.rs b/crates/headroom-proxy/tests/integration_metrics.rsindex f7bf30cdc..cf02f1037 100644--- a/crates/headroom-proxy/tests/integration_metrics.rs+++ b/crates/headroom-proxy/tests/integration_metrics.rs@@ -323,41 +323,103 @@ async fn passthrough_bytes_modified_zero_when_no_compression() {assert_eq!(resp.status(), 200);let scrape = scrape_metrics(&proxy.url()).await;- // The `prometheus` crate v0.13 skips empty MetricVecs entirely- // in `gather()` — neither HELP/TYPE nor rows appear until the- // counter has been incremented with at least one label-set.- // For the "must stay 0" alarm-able metric this absence IS the- // signal: if the counter is silent, no passthrough policy has- // been violated. The PromQL alarm queries- // `rate(proxy_passthrough_bytes_modified_total[5m]) > 0`, which- // is `0` (the metric doesn't exist) by definition.- let any_row = scrape- .lines()- .any(|l| l.starts_with("proxy_passthrough_bytes_modified_total{"));+ // H3 contract: `handle_metrics` force-zeroes every counter /+ // gauge MetricVec under a sentinel `__init__` label tuple on+ // each scrape. That makes HELP/TYPE + a zero row visible from+ // boot so operators have a predictable scrape shape (the+ // pre-H3 behaviour where the family was absent until first+ // emit was confusing — operators would `curl /metrics` on a+ // fresh boot and see nothing). The "must stay 0" alarm+ // semantic is still preserved because the row only carries+ // the sentinel label, not a real production path label.assert!(- !any_row,- "counter should have no rows when nothing modified passthrough; got: {scrape}"+ scrape.contains("# HELP proxy_passthrough_bytes_modified_total"),+ "scrape missing proxy_passthrough_bytes_modified_total HELP on fresh boot (H3): {scrape}");- // The HELP/TYPE descriptor is registered eagerly in- // `handle_metrics`, but `gather()` skips empty families so the- // descriptor only surfaces after the first emit. We assert the- // counter is reachable via the public helper to pin behaviour- // (the helper is what the SSE/handler emit sites call).+ assert!(+ scrape.contains("# TYPE proxy_passthrough_bytes_modified_total counter"),+ "scrape missing proxy_passthrough_bytes_modified_total TYPE on fresh boot (H3): {scrape}"+ );+ let init_row = find_value_with_labels(+ &scrape,+ "proxy_passthrough_bytes_modified_total",+ &[("path", "__init__")],+ )+ .expect("H3 contract: __init__ row must appear on fresh boot");+ assert!(+ (init_row - 0.0).abs() < f64::EPSILON,+ "H3 sentinel __init__ row must read 0 on fresh boot; got {init_row}"+ );+ // The `/v1/messages` path label MUST NOT appear — the+ // dispatcher returned NoCompression and no bytes were+ // mutated, so the alarm did not fire. (Other tests in this+ // suite may have populated rows under their own+ // `path="/integration_test_*"` labels; we filter to the real+ // production path label that THIS test would have produced.)+ let messages_row = find_value_with_labels(+ &scrape,+ "proxy_passthrough_bytes_modified_total",+ &[("path", "/v1/messages")],+ );+ assert!(+ messages_row.is_none(),+ "counter must have no row for /v1/messages when nothing modified passthrough; \+ got value {messages_row:?}"+ );++ // Public helper still works to drive a real-path row.use headroom_proxy::observability::record_passthrough_bytes_modified;record_passthrough_bytes_modified("/integration_test_passthrough_synthetic",- 0,+ 7,"integration_test_request_id_passthrough_v1",);- // After a (zero-delta) increment, the row should appear.let scrape_after_touch = scrape_metrics(&proxy.url()).await;+ let touched = find_value_with_labels(+ &scrape_after_touch,+ "proxy_passthrough_bytes_modified_total",+ &[("path", "/integration_test_passthrough_synthetic")],+ )+ .expect("real-path row must appear after record_passthrough_bytes_modified");assert!(- scrape_after_touch.contains("# HELP proxy_passthrough_bytes_modified_total"),- "scrape missing proxy_passthrough_bytes_modified_total HELP after touch"+ touched >= 7.0,+ "expected ≥7 byte delta after touch; got {touched}");++ proxy.shutdown().await;+}++// ============================================================================+// C2 wire-up: a request that is supposed to passthrough byte-equal+// but whose body bytes change is detected and the alarm fires.+// We exercise the public helper directly because forcing an+// accidental byte mutation at the dispatcher level is itself a+// regression we don't want to provoke deliberately. The helper-+// level test confirms the metric vector + label semantics, and the+// production-path test in `passthrough_bytes_modified_zero_when_no_compression`+// confirms the alarm STAYS silent on the happy path.+// ============================================================================++#[tokio::test]+async fn passthrough_bytes_modified_alarm_fires_with_byte_delta_label() {+ use headroom_proxy::observability::record_passthrough_bytes_modified;++ // Unique path label so this test owns its row.+ const TEST_PATH: &str = "/integration_test_c2_alarm_v1";+ record_passthrough_bytes_modified(TEST_PATH, 42, "integration_test_c2_request_id_v1");+ record_passthrough_bytes_modified(TEST_PATH, 13, "integration_test_c2_request_id_v2");++ let proxy = start_proxy_with("http://127.0.0.1:1", |_| {}).await;+ let scrape = scrape_metrics(&proxy.url()).await;+ let value = find_value_with_labels(+ &scrape,+ "proxy_passthrough_bytes_modified_total",+ &[("path", TEST_PATH)],+ )+ .expect("C2 alarm row must appear");assert!(- scrape_after_touch.contains("# TYPE proxy_passthrough_bytes_modified_total counter"),- "scrape missing proxy_passthrough_bytes_modified_total TYPE after touch"+ (value - 55.0).abs() < f64::EPSILON,+ "C2 alarm increment must reflect summed byte deltas (42 + 13 = 55); got {value}");proxy.shutdown().await;@@ -403,7 +465,9 @@ async fn responses_passthrough_upstream() -> (SocketAddr, tokio::task::JoinHandl}#[tokio::test]-async fn service_tier_logged() {+async fn service_tier_logged_known_value() {+ // C1: spec-defined `service_tier` value lands in its own+ // bucket without going through the `"other"` sentinel.let (addr, _server) = responses_passthrough_upstream().await;let proxy = start_proxy_with(&format!("http://{addr}"), |c| {c.compression_mode = headroom_proxy::config::CompressionMode::Off;@@ -411,11 +475,9 @@ async fn service_tier_logged() {}).await;- // Unique tier value so this test owns its own row.- const TEST_TIER: &str = "integration_test_tier_priority_v1";let body = json!({"model": "gpt-5",- "service_tier": TEST_TIER,+ "service_tier": "priority","input": [{"type":"message","role":"user","content":[{"type":"input_text","text":"hi"}]}]@@ -434,9 +496,9 @@ async fn service_tier_logged() {let tier_count = find_value_with_labels(&scrape,"proxy_service_tier_count_total",- &[("tier", TEST_TIER)],+ &[("tier", "priority")],)- .expect("service tier counter row must appear");+ .expect("service tier counter row must appear under 'priority'");assert!(tier_count >= 1.0,"expected ≥1 service_tier increment; got {tier_count}"@@ -445,6 +507,99 @@ async fn service_tier_logged() {proxy.shutdown().await;}+#[tokio::test]+async fn service_tier_unknown_bucketed_to_other() {+ // C1: a malicious or drifting client sends an unrecognised+ // service_tier value. The bounded-vocabulary validator MUST+ // bucket it to "other" so a malicious client can't blow up+ // the metric vector cardinality.+ let (addr, _server) = responses_passthrough_upstream().await;+ let proxy = start_proxy_with(&format!("http://{addr}"), |c| {+ c.compression_mode = headroom_proxy::config::CompressionMode::Off;+ c.enable_responses_streaming = true;+ })+ .await;++ // Two distinct unknown values — both must bucket to "other".+ for unknown_tier in [+ "integration_test_unknown_tier_alpha_v1",+ "integration_test_unknown_tier_beta_v1",+ ] {+ let body = json!({+ "model": "gpt-5",+ "service_tier": unknown_tier,+ "input": [+ {"type":"message","role":"user","content":[{"type":"input_text","text":"hi"}]}+ ]+ });+ let resp = reqwest::Client::new()+ .post(format!("{}/v1/responses", proxy.url()))+ .header("content-type", "application/json")+ .body(serde_json::to_vec(&body).unwrap())+ .send()+ .await+ .unwrap();+ assert_eq!(resp.status(), 200);+ let _ = resp.bytes().await.unwrap();+ }++ let scrape = scrape_metrics(&proxy.url()).await;+ // Neither raw value may appear as a label — they must be+ // bucketed.+ assert!(+ !scrape.contains("integration_test_unknown_tier_alpha_v1"),+ "raw unknown tier value leaked into metrics (cardinality DoS): {scrape}"+ );+ assert!(+ !scrape.contains("integration_test_unknown_tier_beta_v1"),+ "raw unknown tier value leaked into metrics (cardinality DoS): {scrape}"+ );+ // The "other" bucket must have been incremented at least twice.+ let other_count = find_value_with_labels(+ &scrape,+ "proxy_service_tier_count_total",+ &[("tier", "other")],+ )+ .expect("'other' bucket row must appear");+ assert!(+ other_count >= 2.0,+ "expected ≥2 increments on 'other' bucket (one per unknown tier sent); got {other_count}"+ );++ proxy.shutdown().await;+}++#[test]+fn service_tier_validate_known_returns_canonical_constant() {+ use headroom_proxy::observability::metric_names::service_tier;+ // Spec-defined values pass through verbatim — strict equality+ // against the &'static constants so a typo in the validator+ // surfaces here.+ assert_eq!(service_tier::validate("auto"), service_tier::AUTO);+ assert_eq!(service_tier::validate("default"), service_tier::DEFAULT);+ assert_eq!(service_tier::validate("flex"), service_tier::FLEX);+ assert_eq!(service_tier::validate("on_demand"), service_tier::ON_DEMAND);+ assert_eq!(service_tier::validate("priority"), service_tier::PRIORITY);+ assert_eq!(service_tier::validate("scale"), service_tier::SCALE);+}++#[test]+fn service_tier_validate_unknown_returns_other_sentinel() {+ use headroom_proxy::observability::metric_names::service_tier;+ // C1: anything outside the bounded vocab buckets to OTHER.+ assert_eq!(+ service_tier::validate("nonsense_value"),+ service_tier::OTHER+ );+ assert_eq!(service_tier::validate(""), service_tier::OTHER);+ // Case-sensitive: spec is case-sensitive on these strings.+ assert_eq!(service_tier::validate("PRIORITY"), service_tier::OTHER);+ assert_eq!(service_tier::validate("Auto"), service_tier::OTHER);+ // Extremely long / arbitrary attacker input → still bucketed.+ let attack = "A".repeat(10_000);+ assert_eq!(service_tier::validate(&attack), service_tier::OTHER);+}+// ============================================================================// Test 5: incomplete_status_logged_with_reason// ============================================================================@@ -503,6 +658,73 @@ async fn incomplete_status_logged_with_reason() {// Bonus coverage: rate-limit gauge plumbing via the public helper.// ============================================================================+// ============================================================================+// H1 per-strategy ratio: drive `observe_compression_ratio` twice+// with different strategy names + tokens, and assert each strategy+// row has its OWN sum (not the same aggregate replicated).+// ============================================================================++#[tokio::test]+async fn compression_ratio_per_strategy_does_not_replicate_aggregate() {+ // Pre-H1 the proxy emitted the same `aggregate` ratio per+ // strategy when multiple strategies ran on one body. This test+ // exercises the helper directly with two distinct strategies ++ // distinct ratios so the histogram's _sum lines for each+ // strategy must differ.+ use headroom_proxy::observability::observe_compression_ratio;++ const STRAT_HEAVY: &str = "h1_test_heavy_v1";+ const STRAT_LIGHT: &str = "h1_test_light_v1";+ const CT: &str = "h1_test_content_type_v1";++ // Strategy A: original=1000 tokens → compressed=200 (ratio 0.20).+ observe_compression_ratio(STRAT_HEAVY, CT, 1000, 200);+ // Strategy B: original=1000 tokens → compressed=800 (ratio 0.80).+ observe_compression_ratio(STRAT_LIGHT, CT, 1000, 800);++ let proxy = start_proxy_with("http://127.0.0.1:1", |_| {}).await;+ let scrape = scrape_metrics(&proxy.url()).await;++ let sum_heavy = find_value_with_labels(+ &scrape,+ "proxy_compression_ratio_by_strategy_sum",+ &[("strategy", STRAT_HEAVY), ("content_type", CT)],+ )+ .expect("heavy-strategy sum row");+ let sum_light = find_value_with_labels(+ &scrape,+ "proxy_compression_ratio_by_strategy_sum",+ &[("strategy", STRAT_LIGHT), ("content_type", CT)],+ )+ .expect("light-strategy sum row");+ // The sums must NOT be equal — if they are, the per-strategy+ // wiring regressed to the pre-H1 "emit same aggregate per+ // strategy" behavior.+ assert!(+ (sum_heavy - sum_light).abs() > 1e-9,+ "per-strategy sums are equal: heavy={sum_heavy} light={sum_light} — \+ H1 regression: did we re-emit the aggregate ratio per strategy?"+ );+ // Spot-check the actual ratios.+ assert!(+ sum_heavy < sum_light,+ "heavy strategy ratio (0.20) must be < light strategy ratio (0.80): \+ heavy={sum_heavy} light={sum_light}"+ );++ proxy.shutdown().await;+}++// ============================================================================+// H2 aborted stream: a client disconnect mid-stream closes the+// channel without `message_stop`. Unit-tested in+// `crate::observability::cache_hit_rate::tests` because the+// integration-level approach is flaky against the shared global+// Prometheus registry (other tests in the suite emit on the same+// `provider="anthropic"` label, making delta-based assertions+// non-deterministic).+// ============================================================================+#[tokio::test]async fn rate_limit_snapshot_emits_gauges() {let snap = observability::RateLimitSnapshot {diff --git a/docs/observability.md b/docs/observability.mdindex 9582f1302..33b2f20f6 100644--- a/docs/observability.md+++ b/docs/observability.md@@ -35,6 +35,14 @@ rename catches one file in code review.|------|------|--------|---------|| `proxy_passthrough_bytes_modified_total` | Counter | `path` | Bytes mutated on a passthrough path. **Must stay 0 outside the compression hot path** — any non-zero rate fires the cache-safety alarm. |+The alarm metric is wired in `crates/headroom-proxy/src/proxy.rs`:+when the dispatcher returns `Outcome::NoCompression` or+`Outcome::Passthrough`, the post-dispatcher byte length is compared+to the original buffered length and any delta increments the+counter (by the byte delta) under the request's path label. The+PR-E4 prompt_cache_key injector runs AFTER the alarm check, so its+intentional byte mutations do not trip the alarm.+#### Upstream rate limits| Name | Type | Labels | Purpose |@@ -51,17 +59,29 @@ rename catches one file in code review.| `proxy_service_tier_count_total` | Counter | `tier` | Service-tier distribution observed at the proxy. || `proxy_response_status_count_total` | Counter | `status` | Terminal status distribution (`completed`, `incomplete`, `failed`, `cancelled`, `in_progress`). |-#### Wrap CLI / RTK+#### Wrap CLI / RTK (Python-side)| Name | Type | Labels | Purpose ||------|------|--------|---------|-| `wrap_rtk_invocations_total` | Counter | `tool` | RTK invocations observed via `rtk gain --format json` polling. Driven from the wrap-CLI tail. |+| `wrap_rtk_invocations_total` | Counter | `tool` | RTK invocations observed via the wrap-CLI tail. Surfaced via the Python proxy's `/metrics` exporter; the wrap CLI bumps `headroom.cli.wrap_rtk_metrics.record_rtk_invocation(...)`. |++> **C4 remediation:** This counter is Python-side because RTK is+> wrapped by `headroom wrap` (Python CLI) and the wrap-side tail+> is the natural emit site. The Rust proxy previously held a dead+> counter for this metric; that has been removed.-#### Image log redaction+#### Image log redaction (Python-side)| Name | Type | Labels | Purpose ||------|------|--------|---------|-| `proxy_image_generation_call_log_redacted_total` | Counter | _none_ | Base64-encoded image payloads redacted from request logs. |+| `proxy_image_generation_call_log_redacted_total` | Counter | _none_ | Base64-encoded image payloads redacted from request logs. Driven from `headroom.proxy.request_logger.redactions_total()`. |++> **C3 remediation:** Image redaction is purely a Python-proxy+> operation (the request logger walks JSON and replaces over-+> threshold image payloads with placeholders). The counter lives+> Python-side so we have one source of truth instead of two. The+> Rust proxy previously held a dead counter for this metric; that+> has been removed.## How to query@@ -71,32 +91,155 @@ The proxy renders Prometheus text-format on `GET /metrics`:curl -s http://127.0.0.1:8787/metrics```-Common PromQL queries:+### Phase H canary gate++The canary script that decides "ship Rust, retire Python" uses+**all four** of these queries against `proxy_cache_hit_rate_per_session`+to confirm parity vs the Python baseline. A single percentile is+not enough — a regression that only shows up at the tail (a small+class of long sessions losing cache hits) would slip through a+median-only check.```promql-# Phase H canary gate — cache hit rate parity vs Python proxy.-histogram_quantile(0.50, sum by (le) (rate(proxy_cache_hit_rate_per_session_bucket[5m])))+# p50, p95, p99 of cache hit rate over the last 5 minutes, per provider.+histogram_quantile(0.50, sum by (provider, le) (rate(proxy_cache_hit_rate_per_session_bucket{provider!="__init__"}[5m])))+histogram_quantile(0.95, sum by (provider, le) (rate(proxy_cache_hit_rate_per_session_bucket{provider!="__init__"}[5m])))+histogram_quantile(0.99, sum by (provider, le) (rate(proxy_cache_hit_rate_per_session_bucket{provider!="__init__"}[5m])))++# Mean cache hit rate over the last 5 minutes, per provider. The+# `sum / count` form is the cleanest "average without a quantile"+# query and is what the Python baseline reports.+sum by (provider) (rate(proxy_cache_hit_rate_per_session_sum{provider!="__init__"}[5m]))+ /+sum by (provider) (rate(proxy_cache_hit_rate_per_session_count{provider!="__init__"}[5m]))+```++The canary fails if ANY of `p50`, `p95`, `p99`, or `mean` regresses+below the Python baseline for any provider over the canary window.-# Cache-safety alarm. Should always be 0.-sum(rate(proxy_passthrough_bytes_modified_total[5m]))+### Other common queries-# Aggregate compression value by strategy.-histogram_quantile(0.50, sum by (strategy, le) (rate(proxy_compression_ratio_by_strategy_bucket[1h])))+```promql+# Cache-safety alarm. Should always be 0 (post-`__init__` row).+sum(rate(proxy_passthrough_bytes_modified_total{path!="__init__"}[5m]))++# Per-strategy compression value at p50 (post-H1 fix: each strategy+# reports its own before/after; pre-fix this was the same aggregate+# ratio repeated per strategy).+histogram_quantile(0.50, sum by (strategy, le) (rate(proxy_compression_ratio_by_strategy_bucket{strategy!="__init__"}[1h])))++# Per-strategy compression value at p95 and p99 (catch outlier+# strategies that fail to shrink at the tail).+histogram_quantile(0.95, sum by (strategy, le) (rate(proxy_compression_ratio_by_strategy_bucket{strategy!="__init__"}[1h])))+histogram_quantile(0.99, sum by (strategy, le) (rate(proxy_compression_ratio_by_strategy_bucket{strategy!="__init__"}[1h])))++# Strategies that ran but failed the token-check (compressor ran+# but its output was not strictly smaller, so the original was+# kept). High rate here means the compressor needs tuning.+sum by (strategy) (rate(proxy_compression_rejected_by_token_check_total{strategy!="__init__"}[1h]))# Upstream rate-limit headroom (smaller = closer to throttle).proxy_rate_limit_remaining_tokens{provider="anthropic"}++# RTK invocation rate (Python-side).+sum by (tool) (rate(wrap_rtk_invocations_total{tool!="__init__"}[5m]))++# Image-redaction rate (Python-side).+rate(proxy_image_generation_call_log_redacted_total[5m])```+All queries above include a `{... != "__init__"}` filter so the+sentinel zero-rows the boot-touch contract emits do not skew the+result. See "Wiring → H3 force-zero" below.+## WiringEvery metric registration is `OnceLock`-backed and lazy: the firstcall to a `*_counter()` / `*_gauge()` / `*_histogram()` helperregisters the family with the shared registry. `handle_metrics`-force-touches every Phase G PR-G3 family before scraping so the-descriptors are reachable from `/metrics` on a fresh boot — note that-the `prometheus` crate v0.13 skips empty MetricVecs from `gather()`,-so HELP/TYPE only surface once a family has been incremented at-least once.+force-touches every Phase G PR-G3 family before scraping.++### H3 force-zero++The `prometheus` crate v0.13 skips empty MetricVecs from `gather()`+entirely — neither HELP/TYPE lines nor rows appear until the+family has been incremented at least once with a label tuple.+Operators expect to see the catalogue from boot, so+`handle_metrics` increments each counter / gauge MetricVec by 0+under a sentinel `__init__` label tuple before the first scrape.+HELP/TYPE then surface from boot and dashboards/alarms see a+predictable scrape shape.++Counters with the `__init__` label increment by 0, so the+alarm-able "must stay 0" semantic of+`proxy_passthrough_bytes_modified_total` is preserved (the family+becomes visible, the rate stays 0). PromQL queries should filter+`{... != "__init__"}` so the sentinel rows are excluded from+aggregations (the catalogue above does this).++Histograms are NOT force-zeroed: a synthetic `observe(0.0)` would+contribute a real sample to the per-label distribution and pollute+percentile readings. The two histogram families+(`proxy_cache_hit_rate_per_session` and+`proxy_compression_ratio_by_strategy`) only surface in the scrape+after the first real session, by design.++### H4 prometheus crate version pin++The H3 contract above relies on the `prometheus` crate's v0.13+`gather()` semantics — empty MetricVec families are omitted from+the scrape. **This is implementation-defined behaviour.** If+`crates/headroom-proxy/Cargo.toml` ever bumps the `prometheus`+dependency, retest the alarm contract:++1. Start a fresh proxy.+2. `curl /metrics` and confirm every counter / gauge family has+ HELP/TYPE + an `__init__` row.+3. Confirm histograms (`*_cache_hit_rate_per_session`,+ `*_compression_ratio_by_strategy`) DO NOT appear (no+ `observe()` calls yet).+4. Drive one cache-hit session, scrape again, confirm histograms+ now appear.+5. Confirm `passthrough_bytes_modified_total` stays at 0 across+ passthrough requests.++The crate version is pinned exactly (`= "0.13.4"`, no caret) in+`Cargo.toml` precisely so a silent semver bump cannot break the+contract without a code-review trigger.++### C2 alarm wiring++`proxy_passthrough_bytes_modified_total` fires from `proxy.rs` when+a dispatcher arm that promised byte-equal passthrough+(`Outcome::NoCompression` or `Outcome::Passthrough`) produces a+final body of a different byte length. The check runs BEFORE the+PR-E4 prompt_cache_key injector so the injector's intentional byte+mutations do not trip the alarm.++### H1 per-strategy ratio wiring++`proxy_compression_ratio_by_strategy` samples one observation per+strategy using the strategy's OWN before/after token counts+(plumbed through `Outcome::Compressed.per_strategy_tokens` from+the manifest in `live_zone_anthropic` / `live_zone_openai` /+`live_zone_responses`). Pre-H1 the same aggregate ratio was+emitted per strategy when multiple strategies ran on one body,+making Phase H per-strategy dashboards read garbage.++### H2 aborted-stream gate++The `proxy_cache_hit_rate_per_session` histogram observes ONLY+when the SSE stream completed:++* Anthropic: `state.status == StreamStatus::MessageStop` after the+ channel closes.+* OpenAI Chat: `state.usage.is_some()` (the final usage chunk only+ arrives at stream completion).+* OpenAI Responses: `state.terminal_status().is_some()`.++A client disconnect mid-stream closes the channel without setting+the terminal flag — under H2 we log + skip rather than observe a+garbage half-stream sample.## Cardinality discipline@@ -107,8 +250,18 @@ Every label vocabulary is bounded by code, not customer input:- `provider`: 3 values (`anthropic`, `openai_chat`, `openai_responses`).- `strategy`: `&'static str` from the compressor's `BlockAction::Compressed`.- `content_type`: `&'static str` from `headroom_core::transforms::ContentType`.-- `tier`: bounded by OpenAI Responses spec.+- `tier`: validated through+ `crate::observability::metric_names::service_tier::validate(raw: &str)`.+ Returns one of `{auto, default, flex, on_demand, priority, scale}`+ or the sentinel `"other"` for anything else. **The raw inbound+ value is never used as a label.** A malicious client posting+ `{"service_tier":"<random>"}` per request gets bucketed to+ `"other"` and a `tracing::warn!` is emitted so wire-format drift+ surfaces loudly in logs.- `status`: 5-variant enum.+- `tool` (Python-side `wrap_rtk_invocations_total`): bounded by the+ set of tools the wrap CLI rewrites, captured by+ `headroom.cli.wrap_rtk_metrics`.There is no code path where a malicious client can drive labelcardinality unbounded.diff --git a/headroom/cli/wrap_rtk_metrics.py b/headroom/cli/wrap_rtk_metrics.pynew file mode 100644index 000000000..64c788788--- /dev/null+++ b/headroom/cli/wrap_rtk_metrics.py@@ -0,0 +1,73 @@+"""RTK invocation metrics for the wrap CLI.++Phase G PR-G3 remediation (C4): RTK lives wrap-side, not proxy-side+(see ``docs/rtk-architecture.md``). The wrap CLI tails+``rtk gain --format json`` and bumps a process-local counter keyed+by rewritten command name (`git`, `ls`, `cargo`, ...). The Python+proxy's ``/metrics`` endpoint then surfaces the counter as+``wrap_rtk_invocations_total{tool=...}`` for fleet-wide scrape.++The counter primitives live here (not in ``wrap.py``) so the+proxy's prometheus exporter can import them without dragging in the+full ``wrap.py`` module — that module owns subprocess-level CLI+spawning and is heavyweight to import at proxy startup.++Per realignment build-constraint "no silent fallbacks":+``record_rtk_invocation`` raises on a non-string tool name rather+than coercing; a caller passing the wrong type is a bug, not a+runtime fallback condition.+"""++from __future__ import annotations++import threading+from collections import defaultdict+from collections.abc import Mapping++# Module-level counter — process-local. Multiple worker processes+# (uvicorn workers) each maintain their own; the Python proxy already+# documents this in ``docs/observability.md``. Reset is exposed for+# tests; production code never reaches for it.+_rtk_invocation_counts: dict[str, int] = defaultdict(int)+_lock = threading.Lock()+++def record_rtk_invocation(tool: str, delta: int = 1) -> None:+ """Record one (or `delta`) RTK invocation(s) for the given tool.++ `tool` is the rewritten command name as observed in the+ ``rtk gain --format json`` output (e.g. ``"git"``, ``"ls"``,+ ``"cargo"``). The counter is keyed verbatim.++ `delta` defaults to 1 for the common "one invocation seen" path+ but accepts arbitrary positive deltas so the wrap tail can bump+ by a JSON-reported batch count.++ Raises:+ TypeError: if `tool` is not a `str` or `delta` is not an `int`.+ ValueError: if `delta` is negative.+ """+ if not isinstance(tool, str):+ raise TypeError(f"tool must be a str, got {type(tool).__name__}")+ if not isinstance(delta, int):+ raise TypeError(f"delta must be an int, got {type(delta).__name__}")+ if delta < 0:+ raise ValueError(f"delta must be non-negative, got {delta}")+ with _lock:+ _rtk_invocation_counts[tool] += delta+++def rtk_invocation_counts() -> Mapping[str, int]:+ """Return a snapshot of the current invocation counts.++ Returns a plain dict (not the defaultdict) so callers cannot+ accidentally pollute the counter map by reading absent keys.+ """+ with _lock:+ return dict(_rtk_invocation_counts)+++def reset_rtk_invocations() -> None:+ """Reset the counter map. Test-only — never called from production."""+ with _lock:+ _rtk_invocation_counts.clear()diff --git a/headroom/proxy/prometheus_metrics.py b/headroom/proxy/prometheus_metrics.pyindex 600b485f8..27ad9fb1c 100644--- a/headroom/proxy/prometheus_metrics.py+++ b/headroom/proxy/prometheus_metrics.py@@ -1215,4 +1215,56 @@ class PrometheusMetrics:)lines.append("")+ # Phase G PR-G3 remediation (C3): image-redacted counter+ # lives Python-side because base64 redaction is purely a+ # Python-proxy concern (request_logger.py). The Rust+ # proxy previously held a dead counter for this; that's+ # been removed in favour of this Python export.+ #+ # The counter is read at scrape-time from the module-+ # level redaction tracker rather than mirrored into the+ # PrometheusMetrics instance, so we never lose a count+ # to ordering between RequestLogger setup and metrics+ # init.+ from headroom.proxy.request_logger import redactions_total++ _append_metric(+ lines,+ name="proxy_image_generation_call_log_redacted_total",+ metric_type="counter",+ help_text=(+ "Count of base64-encoded image payloads redacted from request "+ "logs by the Python proxy's request logger"+ ),+ value=redactions_total(),+ )++ # Phase G PR-G3 remediation (C4): RTK invocations counter+ # also lives Python-side. RTK is wrapped by the+ # `headroom wrap` CLI (headroom.cli.wrap); the proxy+ # observes invocation counts via a process-local tracker+ # the wrap tail bumps. The Rust proxy previously held a+ # dead counter for this; that's been removed.+ from headroom.cli.wrap_rtk_metrics import rtk_invocation_counts++ counts = rtk_invocation_counts()+ lines.extend(+ [+ "# HELP wrap_rtk_invocations_total RTK invocations observed via the wrap CLI tail",+ "# TYPE wrap_rtk_invocations_total counter",+ ]+ )+ if not counts:+ # Emit a zero-row under the sentinel tool name so+ # the family advertises HELP/TYPE on a fresh boot+ # and dashboards can probe it before any RTK+ # invocation has happened. Matches the Rust side's+ # H3 force-zero contract.+ lines.append('wrap_rtk_invocations_total{tool="__init__"} 0')+ else:+ for tool, count in counts.items():+ safe_tool = _escape_label_value(str(tool))+ lines.append(f'wrap_rtk_invocations_total{{tool="{safe_tool}"}} {count}')+ lines.append("")+return "\n".join(lines)diff --git a/headroom/proxy/request_logger.py b/headroom/proxy/request_logger.pyindex a29b49871..c367b6f4d 100644--- a/headroom/proxy/request_logger.py+++ b/headroom/proxy/request_logger.py@@ -8,6 +8,15 @@ Phase G PR-G3 (P4-45): base64-encoded image payloads in the``request_messages`` / ``response_content`` are redacted beforewrite to keep request logs small. Multi-MB base64 strings wouldotherwise saturate the JSONL log and the in-memory deque.++Remediation (M2, M5): the redactor now ONLY fires inside known+image-bearing JSON paths or against strings that carry an explicit+``data:image/...;base64,`` URL prefix. The earlier "density+heuristic" over-fired on encrypted blobs, signed tokens, minified+JSON, and tool outputs. The replacement placeholder now reports+the UTF-8 byte length under a ``bytes=`` label (was character+length; for the ASCII base64 alphabet the two happen to coincide+but the label is now accurate for any future Unicode payload)."""from __future__ import annotations@@ -19,6 +28,7 @@ from collections import dequefrom collections.abc import Mapping, Sequencefrom dataclasses import asdictfrom pathlib import Path+from threading import Lockfrom typing import TYPE_CHECKING, Anyif TYPE_CHECKING:@@ -40,74 +50,124 @@ IMAGE_BASE64_REDACT_THRESHOLD_BYTES = 1024# Phase G PR-G3 — replacement-marker format. Operators can grep the# JSONL for ``<image:base64-redacted`` to count the redactions; the# byte count keeps cost attribution honest even after redaction.+# M5: ``bytes=`` is the UTF-8 byte length, not the character count.IMAGE_BASE64_REPLACEMENT_TEMPLATE = "<image:base64-redacted bytes={n}>"-# Constants for log redaction counter export (Prometheus). The Rust-# proxy owns the canonical metric; the Python side increments a-# best-effort module-level counter so tests and the in-process-# ``/stats`` endpoint can read back the redaction rate.+# M2: JSON field names that carry image payloads in either the+# Anthropic or OpenAI shapes. Strings reached via one of these key+# names (at any depth) are eligible for the redaction heuristic.+# Anything OUTSIDE these paths is left untouched even if it looks+# base64-shaped — encrypted blobs, signed tokens, minified JSON,+# tool outputs all live elsewhere and stay verbatim.+IMAGE_BEARING_FIELD_NAMES: frozenset[str] = frozenset(+ {+ # Anthropic image-block shape: ``{"type":"image","source":{"type":"base64","data":"..."}}``.+ "data",+ # OpenAI vision shape: ``{"type":"image_url","image_url":{"url":"data:image/..."}}``.+ "url",+ # OpenAI Responses input_image: ``{"type":"input_image","image_url":"..."}``+ # — string-valued directly under the key (not nested).+ "image_url",+ # Some SDKs put the URL under ``image`` directly. Tolerated.+ "image",+ # Anthropic vision blocks sometimes wrap under ``source.data``;+ # ``source`` is a container, not a string field, so it doesn't+ # need to be in this set, but the data string itself is keyed+ # by ``data`` (already above).+ }+)++# M2: explicit data-URL MIME prefix. A string starting with this+# prefix is always treated as an image payload, regardless of where+# it lives in the JSON — operators occasionally embed data URLs in+# arbitrary fields and we want those redacted to keep logs small.+_DATA_IMAGE_URL_PREFIX = "data:image/"+++# Constants for log redaction counter export (Prometheus). The+# Python proxy's ``/metrics`` exporter surfaces+# ``proxy_image_generation_call_log_redacted_total`` from this+# module-level counter. C3 remediation: the Rust proxy previously+# held a dead counter; that's been removed in favour of this+# Python-side counter, which is the natural owner._redactions_total: int = 0+_redactions_lock = Lock()def redactions_total() -> int:"""Return the running count of base64 redactions performed.- Exposed for unit tests + the legacy Python ``/stats`` endpoint.- The canonical observability surface is the Rust proxy's- ``proxy_image_generation_call_log_redacted_total`` metric.+ Exposed for unit tests, the legacy Python ``/stats`` endpoint,+ and the Prometheus exporter+ (``proxy_image_generation_call_log_redacted_total``)."""- return _redactions_total+ with _redactions_lock:+ return _redactions_total-def _looks_like_base64_image(value: str) -> bool:- """Heuristic: does ``value`` look like a base64-encoded image?+def _is_base64_image_payload(value: str) -> bool:+ """Return True if ``value`` is an over-threshold base64 image.- Two patterns we recognise:+ Per M2 remediation the prior bare-base64 density heuristic+ over-fired on non-image content (encrypted blobs, signed+ tokens, minified JSON, tool outputs). We now only consider a+ string an image payload when EITHER:- * Raw base64 over the threshold (Anthropic ``source.data`` shape).- * ``data:image/<subtype>;base64,<payload>`` data URLs (OpenAI- vision shape). The ``;base64,`` substring is the load-bearing- signal — any data URL with that segment over the threshold gets- redacted, even if the MIME type isn't ``image/...`` (because- the cost-of-logging is paid the same way).+ 1. It starts with ``data:image/`` (an explicit data URL),+ OR+ 2. The caller has already established the string lives inside+ an image-bearing JSON path (see ``IMAGE_BEARING_FIELD_NAMES``)+ AND the string itself is over the byte threshold.- Returns ``False`` for short strings (under the threshold) and for- any non-string value. Per realignment build-constraint "no- regexes", we use prefix/substring checks instead of pattern- matching.+ Case (2) is decided by the caller (``_redact_value``) which+ threads ``in_image_path`` through the recursion; this helper+ handles case (1) on its own."""if not isinstance(value, str):return Falseif len(value) < IMAGE_BASE64_REDACT_THRESHOLD_BYTES:return False- if value.startswith("data:") and ";base64," in value[:64]:- return True- # Bare base64 payload: heuristic — over-threshold string with no- # whitespace and a high alpha-num+/+= density. We sample the first- # 256 bytes for speed (the full string can be megabytes).- head = value[:256]- base64_chars = set("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=")- matches = sum(1 for ch in head if ch in base64_chars)- return matches / len(head) > 0.95---def _redact_value(value: Any) -> Any:+ return value.startswith(_DATA_IMAGE_URL_PREFIX)+++def _redact_value(value: Any, *, in_image_path: bool = False) -> Any:"""Recursively redact base64-image payloads in a JSON-ish value.Returns a new structure with any over-threshold base64 stringreplaced by the placeholder. Non-string, non-container valuespass through unchanged.++ ``in_image_path`` is True when the caller reached this value+ via one of the ``IMAGE_BEARING_FIELD_NAMES`` keys; once inside+ an image-bearing field, any over-threshold string is treated+ as an image payload (M2: prevents redaction of unrelated+ base64-shaped content outside known image fields)."""global _redactions_totalif isinstance(value, str):- if _looks_like_base64_image(value):- _redactions_total += 1- return IMAGE_BASE64_REPLACEMENT_TEMPLATE.format(n=len(value))+ # Always-redact: explicit data URL, regardless of path.+ # Also redact when the caller signalled image-bearing path+ # AND the string is over threshold (no density check — the+ # path tells us it's an image).+ should_redact = _is_base64_image_payload(value) or (+ in_image_path and len(value) >= IMAGE_BASE64_REDACT_THRESHOLD_BYTES+ )+ if should_redact:+ with _redactions_lock:+ _redactions_total += 1+ byte_len = len(value.encode("utf-8"))+ return IMAGE_BASE64_REPLACEMENT_TEMPLATE.format(n=byte_len)return valueif isinstance(value, Mapping):- return {k: _redact_value(v) for k, v in value.items()}+ return {+ k: _redact_value(+ v,+ in_image_path=(k in IMAGE_BEARING_FIELD_NAMES),+ )+ for k, v in value.items()+ }if isinstance(value, Sequence) and not isinstance(value, str | bytes | bytearray):- return [_redact_value(item) for item in value]+ return [_redact_value(item, in_image_path=in_image_path) for item in value]return value@@ -118,7 +178,7 @@ def redact_image_base64(payload: Any) -> Any:over-threshold base64 string with a size-only placeholder.Idempotent — applying twice yields the same structure."""- return _redact_value(payload)+ return _redact_value(payload, in_image_path=False)class RequestLogger:diff --git a/tests/test_cli/test_wrap_rtk_metrics.py b/tests/test_cli/test_wrap_rtk_metrics.pynew file mode 100644index 000000000..f864e52b7--- /dev/null+++ b/tests/test_cli/test_wrap_rtk_metrics.py@@ -0,0 +1,113 @@+"""Phase G PR-G3 remediation (C4) — wrap-CLI RTK metrics primitive.++The Rust proxy previously held a dead `wrap_rtk_invocations_total`+counter. C4 remediation moved it Python-side because the wrap CLI+(headroom.cli.wrap) is where RTK invocations are actually counted.+These tests cover the counter primitives in isolation.+"""++from __future__ import annotations++import threading++import pytest++from headroom.cli.wrap_rtk_metrics import (+ record_rtk_invocation,+ reset_rtk_invocations,+ rtk_invocation_counts,+)++[email protected](autouse=True)+def _reset_between_tests():+ """Reset the module-level counter map between tests so each+ test owns a clean slate."""+ reset_rtk_invocations()+ yield+ reset_rtk_invocations()+++def test_record_increments_default_delta_one():+ record_rtk_invocation("git")+ counts = rtk_invocation_counts()+ assert counts == {"git": 1}+++def test_record_accumulates_per_tool():+ record_rtk_invocation("git")+ record_rtk_invocation("git")+ record_rtk_invocation("ls")+ record_rtk_invocation("cargo")+ record_rtk_invocation("cargo")+ record_rtk_invocation("cargo")+ counts = rtk_invocation_counts()+ assert counts == {"git": 2, "ls": 1, "cargo": 3}+++def test_record_with_explicit_delta():+ record_rtk_invocation("git", delta=5)+ record_rtk_invocation("git", delta=2)+ counts = rtk_invocation_counts()+ assert counts == {"git": 7}+++def test_record_zero_delta_is_noop_record():+ # delta=0 is legal — caller may want to "touch" the counter to+ # ensure the key exists before later increments.+ record_rtk_invocation("git", delta=0)+ counts = rtk_invocation_counts()+ assert counts == {"git": 0}+++def test_record_rejects_negative_delta():+ with pytest.raises(ValueError, match="must be non-negative"):+ record_rtk_invocation("git", delta=-1)+++def test_record_rejects_non_string_tool():+ with pytest.raises(TypeError, match="tool must be a str"):+ record_rtk_invocation(123, delta=1) # type: ignore[arg-type]+++def test_record_rejects_non_int_delta():+ with pytest.raises(TypeError, match="delta must be an int"):+ record_rtk_invocation("git", delta="1") # type: ignore[arg-type]+++def test_counts_returns_snapshot_not_view():+ # The returned mapping must be a plain dict copy, not the+ # internal defaultdict — otherwise callers could pollute the+ # counter map by reading absent keys.+ record_rtk_invocation("git")+ counts = rtk_invocation_counts()+ # Reading a key that's not present must not add it to the+ # internal map.+ _ = counts.get("nonexistent_tool", 0)+ counts2 = rtk_invocation_counts()+ assert "nonexistent_tool" not in counts2+++def test_thread_safe_concurrent_increments():+ # 10 threads each bumping `git` 100 times: final count must be+ # exactly 1000. The threading.Lock guards the dict update so+ # races are impossible.+ def worker():+ for _ in range(100):+ record_rtk_invocation("git")++ threads = [threading.Thread(target=worker) for _ in range(10)]+ for t in threads:+ t.start()+ for t in threads:+ t.join()+ counts = rtk_invocation_counts()+ assert counts == {"git": 1000}+++def test_reset_clears_counts():+ record_rtk_invocation("git", delta=42)+ record_rtk_invocation("ls", delta=7)+ assert rtk_invocation_counts() != {}+ reset_rtk_invocations()+ assert rtk_invocation_counts() == {}diff --git a/tests/test_image_log_redaction.py b/tests/test_image_log_redaction.pyindex cd6bf9c60..b829322fc 100644--- a/tests/test_image_log_redaction.py+++ b/tests/test_image_log_redaction.py@@ -124,7 +124,8 @@ def test_data_url_redacted():def test_redact_idempotent():"""Applying redaction twice yields the same structure — theplaceholder is short enough to stay below the threshold so the- second pass is a no-op."""+ second pass is a no-op. The ``data`` key is one of the+ image-bearing field names so a big string inside redacts."""big = _big_base64(IMAGE_BASE64_REDACT_THRESHOLD_BYTES * 3)once = redact_image_base64({"data": big})twice = redact_image_base64(once)@@ -165,15 +166,81 @@ def test_logger_writes_redacted_payload_to_jsonl():assert data_field == IMAGE_BASE64_REPLACEMENT_TEMPLATE.format(n=len(big))-def test_response_content_redacted():- """``response_content`` (a string field) is redacted in-place on- the deque entry."""+def test_response_content_bare_base64_passes_through():+ """M2 remediation: a bare base64-shaped string in+ ``response_content`` is NOT redacted. The earlier "density+ heuristic" over-fired on encrypted blobs, signed tokens,+ minified JSON, and tool outputs. The new contract: only+ redact strings inside known image-bearing JSON paths OR+ strings starting with ``data:image/``."""logger = RequestLogger(log_file=None, log_full_messages=True)big = _big_base64(IMAGE_BASE64_REDACT_THRESHOLD_BYTES * 3)entry = _make_request_log(response_content=big)logger.log(entry)recent = logger.get_recent_with_messages(n=1)- assert recent[0]["response_content"] == IMAGE_BASE64_REPLACEMENT_TEMPLATE.format(n=len(big))+ # Verbatim — no redaction applied.+ assert recent[0]["response_content"] == big+++def test_response_content_data_image_url_redacted():+ """When ``response_content`` does start with ``data:image/`` —+ e.g. a tool wrote an image back via a data URL — redaction+ still fires."""+ logger = RequestLogger(log_file=None, log_full_messages=True)+ payload = _big_base64(IMAGE_BASE64_REDACT_THRESHOLD_BYTES * 2)+ data_url = f"data:image/png;base64,{payload}"+ entry = _make_request_log(response_content=data_url)+ logger.log(entry)+ recent = logger.get_recent_with_messages(n=1)+ assert recent[0]["response_content"] == IMAGE_BASE64_REPLACEMENT_TEMPLATE.format(+ n=len(data_url)+ )+++def test_non_image_path_base64_passes_through():+ """M2: a big base64-shaped string at a non-image-bearing key+ (e.g. an encrypted blob under ``signature`` or a tool output+ under ``arguments``) is NOT redacted."""+ big = _big_base64(IMAGE_BASE64_REDACT_THRESHOLD_BYTES * 2)+ payload = {+ "tool_use_id": "tool_xyz",+ "signature": big, # NOT an image-bearing key+ "arguments": big, # NOT an image-bearing key+ }+ redacted = redact_image_base64(payload)+ assert redacted["signature"] == big+ assert redacted["arguments"] == big+++def test_image_path_redacts_without_density_check():+ """M2: once inside an image-bearing JSON path (e.g.+ ``source.data``), a sufficiently-long string is redacted+ regardless of its character density. Real images may not be+ base64 only — they may be webp / avif transcoded with+ different alphabets — but we still want them redacted to+ keep logs bounded."""+ big = "x" * (IMAGE_BASE64_REDACT_THRESHOLD_BYTES * 2) # NOT base64+ payload = {"source": {"type": "base64", "data": big}}+ redacted = redact_image_base64(payload)+ expected_bytes = len(big.encode("utf-8"))+ assert redacted["source"]["data"] == IMAGE_BASE64_REPLACEMENT_TEMPLATE.format(n=expected_bytes)+++def test_byte_count_label_is_utf8_bytes_not_chars():+ """M5: the ``bytes=`` label is the UTF-8 byte length of the+ redacted string, not the character count. For ASCII base64+ payloads the two coincide (so existing tests still pass), but+ a non-ASCII string under an image-bearing key reports byte+ length faithfully."""+ # 3-byte UTF-8 character ('€' = U+20AC) repeated; the+ # character count is half the byte count.+ chars = "€" * (IMAGE_BASE64_REDACT_THRESHOLD_BYTES + 1)+ payload = {"data": chars}+ redacted = redact_image_base64(payload)+ char_count = len(chars)+ byte_count = len(chars.encode("utf-8"))+ assert byte_count == 3 * char_count+ assert redacted["data"] == IMAGE_BASE64_REPLACEMENT_TEMPLATE.format(n=byte_count)def test_redactions_counter_advances():