Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

feat(beacon): schema v2 — routing/compression training signal, gzip transport (#3253)

Tejas Chopra67eb910e38b9
diff --git a/deploy/beacon/sample-event.json b/deploy/beacon/sample-event.json
index 4bacb0ebc..998eb6e9e 100644
--- a/deploy/beacon/sample-event.json
+++ b/deploy/beacon/sample-event.json
@@ -12,7 +12,7 @@
{
"key": "service.version",
"value": {
- "stringValue": "0.34.0"
+ "stringValue": "0.37.0"
}
},
{
@@ -21,6 +21,18 @@
"stringValue": "00000000000000000000000000000000"
}
},
+ {
+ "key": "headroom.install_mode",
+ "value": {
+ "stringValue": "pip"
+ }
+ },
+ {
+ "key": "headroom.stack",
+ "value": {
+ "stringValue": "proxy"
+ }
+ },
{
"key": "os.type",
"value": {
@@ -42,14 +54,14 @@
},
"logRecords": [
{
- "timeUnixNano": "1785731364402434048",
+ "timeUnixNano": "1787703693355874048",
"body": {
"kvlistValue": {
"values": [
{
"key": "schema_version",
"value": {
- "intValue": "1"
+ "intValue": "2"
}
},
{
@@ -60,7 +72,7 @@
{
"key": "id",
"value": {
- "stringValue": "sample0000000001"
+ "stringValue": "4996e02d2b7cafe7"
}
},
{
@@ -72,25 +84,25 @@
{
"key": "duration_s",
"value": {
- "intValue": "4210"
+ "intValue": "135"
}
},
{
"key": "turns",
"value": {
- "intValue": "47"
+ "intValue": "4"
}
},
{
"key": "ended",
"value": {
- "stringValue": "active"
+ "stringValue": "shutdown"
}
},
{
"key": "final",
"value": {
- "boolValue": false
+ "boolValue": true
}
}
]
@@ -105,55 +117,55 @@
{
"key": "original",
"value": {
- "intValue": "890000"
+ "intValue": "480000"
}
},
{
"key": "attempted",
"value": {
- "intValue": "410000"
+ "intValue": "160000"
}
},
{
"key": "input",
"value": {
- "intValue": "570000"
+ "intValue": "360000"
}
},
{
"key": "output",
"value": {
- "intValue": "41000"
+ "intValue": "4800"
}
},
{
"key": "saved",
"value": {
- "intValue": "320000"
+ "intValue": "120000"
}
},
{
"key": "tool_saved",
"value": {
- "intValue": "48000"
+ "intValue": "0"
}
},
{
"key": "cache_read",
"value": {
- "intValue": "210000"
+ "intValue": "240000"
}
},
{
"key": "cache_write",
"value": {
- "intValue": "30000"
+ "intValue": "32000"
}
},
{
"key": "uncached",
"value": {
- "intValue": "650000"
+ "intValue": "88000"
}
}
]
@@ -168,31 +180,43 @@
{
"key": "saved_pct",
"value": {
- "doubleValue": 35.96
+ "doubleValue": 25.0
}
},
{
"key": "eligible_pct",
"value": {
- "doubleValue": 46.07
+ "doubleValue": 33.33
}
},
{
"key": "yield_pct",
"value": {
- "doubleValue": 78.05
+ "doubleValue": 75.0
+ }
+ },
+ {
+ "key": "all_layers_saved_pct",
+ "value": {
+ "doubleValue": 25.0
+ }
+ },
+ {
+ "key": "all_layers_yield_pct",
+ "value": {
+ "doubleValue": 75.0
}
},
{
"key": "cache_read_pct",
"value": {
- "doubleValue": 23.6
+ "doubleValue": 50.0
}
},
{
"key": "overhead_pct",
"value": {
- "doubleValue": 1.96
+ "doubleValue": 1.14
}
}
]
@@ -212,23 +236,43 @@
{
"key": "crush",
"value": {
- "intValue": "47"
+ "intValue": "4"
}
}
]
}
}
},
+ {
+ "key": "by_strategy",
+ "value": {
+ "arrayValue": {
+ "values": []
+ }
+ }
+ },
{
"key": "overhead_ms_total",
"value": {
- "intValue": "1840"
+ "doubleValue": 192.0
}
},
{
"key": "latency_ms_total",
"value": {
- "intValue": "94000"
+ "doubleValue": 16800.0
+ }
+ },
+ {
+ "key": "overhead_ms_per_turn",
+ "value": {
+ "doubleValue": 48.0
+ }
+ },
+ {
+ "key": "latency_ms_per_turn",
+ "value": {
+ "doubleValue": 4200.0
}
},
{
@@ -240,7 +284,7 @@
{
"key": "response_cache_hits",
"value": {
- "intValue": "3"
+ "intValue": "0"
}
}
]
@@ -255,6 +299,21 @@
}
}
},
+ {
+ "key": "sources",
+ "value": {
+ "kvlistValue": {
+ "values": [
+ {
+ "key": "proxy",
+ "value": {
+ "intValue": "4"
+ }
+ }
+ ]
+ }
+ }
+ },
{
"key": "providers",
"value": {
@@ -273,7 +332,7 @@
"arrayValue": {
"values": [
{
- "stringValue": "claude-sonnet-4-5-20250929"
+ "stringValue": "claude-sonnet-5"
}
]
}
@@ -282,23 +341,751 @@
{
"key": "failures",
"value": {
- "intValue": "2"
+ "intValue": "0"
}
},
{
"key": "failure_statuses",
+ "value": {
+ "kvlistValue": {
+ "values": []
+ }
+ }
+ },
+ {
+ "key": "quality",
"value": {
"kvlistValue": {
"values": [
{
- "key": "529",
+ "key": "reread_tokens",
+ "value": {
+ "intValue": "3600"
+ }
+ },
+ {
+ "key": "reread_compressed_tokens",
"value": {
- "intValue": "2"
+ "intValue": "1600"
+ }
+ },
+ {
+ "key": "waste_turns",
+ "value": {
+ "intValue": "4"
+ }
+ },
+ {
+ "key": "waste",
+ "value": {
+ "arrayValue": {
+ "values": []
+ }
}
}
]
}
}
+ },
+ {
+ "key": "hist",
+ "value": {
+ "kvlistValue": {
+ "values": [
+ {
+ "key": "msgs",
+ "value": {
+ "kvlistValue": {
+ "values": [
+ {
+ "key": "edges",
+ "value": {
+ "arrayValue": {
+ "values": [
+ {
+ "intValue": "5"
+ },
+ {
+ "intValue": "20"
+ },
+ {
+ "intValue": "50"
+ },
+ {
+ "intValue": "100"
+ },
+ {
+ "intValue": "250"
+ },
+ {
+ "intValue": "500"
+ }
+ ]
+ }
+ }
+ },
+ {
+ "key": "counts",
+ "value": {
+ "arrayValue": {
+ "values": [
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "4"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ }
+ ]
+ }
+ }
+ }
+ ]
+ }
+ }
+ },
+ {
+ "key": "overhead_ms",
+ "value": {
+ "kvlistValue": {
+ "values": [
+ {
+ "key": "edges",
+ "value": {
+ "arrayValue": {
+ "values": [
+ {
+ "doubleValue": 1.0
+ },
+ {
+ "doubleValue": 5.0
+ },
+ {
+ "doubleValue": 25.0
+ },
+ {
+ "doubleValue": 100.0
+ },
+ {
+ "doubleValue": 500.0
+ },
+ {
+ "doubleValue": 2000.0
+ }
+ ]
+ }
+ }
+ },
+ {
+ "key": "counts",
+ "value": {
+ "arrayValue": {
+ "values": [
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "4"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ }
+ ]
+ }
+ }
+ }
+ ]
+ }
+ }
+ },
+ {
+ "key": "saved_pct",
+ "value": {
+ "kvlistValue": {
+ "values": [
+ {
+ "key": "edges",
+ "value": {
+ "arrayValue": {
+ "values": [
+ {
+ "doubleValue": 0.5
+ },
+ {
+ "doubleValue": 5.0
+ },
+ {
+ "doubleValue": 10.0
+ },
+ {
+ "doubleValue": 20.0
+ },
+ {
+ "doubleValue": 40.0
+ },
+ {
+ "doubleValue": 60.0
+ }
+ ]
+ }
+ }
+ },
+ {
+ "key": "counts",
+ "value": {
+ "arrayValue": {
+ "values": [
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "4"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ }
+ ]
+ }
+ }
+ }
+ ]
+ }
+ }
+ },
+ {
+ "key": "ttfb_ms",
+ "value": {
+ "kvlistValue": {
+ "values": [
+ {
+ "key": "edges",
+ "value": {
+ "arrayValue": {
+ "values": [
+ {
+ "doubleValue": 200.0
+ },
+ {
+ "doubleValue": 500.0
+ },
+ {
+ "doubleValue": 1000.0
+ },
+ {
+ "doubleValue": 2500.0
+ },
+ {
+ "doubleValue": 5000.0
+ },
+ {
+ "doubleValue": 15000.0
+ }
+ ]
+ }
+ }
+ },
+ {
+ "key": "counts",
+ "value": {
+ "arrayValue": {
+ "values": [
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "4"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ }
+ ]
+ }
+ }
+ }
+ ]
+ }
+ }
+ },
+ {
+ "key": "turn_tokens",
+ "value": {
+ "kvlistValue": {
+ "values": [
+ {
+ "key": "edges",
+ "value": {
+ "arrayValue": {
+ "values": [
+ {
+ "intValue": "1000"
+ },
+ {
+ "intValue": "4000"
+ },
+ {
+ "intValue": "16000"
+ },
+ {
+ "intValue": "64000"
+ },
+ {
+ "intValue": "128000"
+ },
+ {
+ "intValue": "256000"
+ }
+ ]
+ }
+ }
+ },
+ {
+ "key": "counts",
+ "value": {
+ "arrayValue": {
+ "values": [
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "4"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ }
+ ]
+ }
+ }
+ }
+ ]
+ }
+ }
+ }
+ ]
+ }
+ }
+ },
+ {
+ "key": "shapes",
+ "value": {
+ "kvlistValue": {
+ "values": [
+ {
+ "key": "by_content",
+ "value": {
+ "arrayValue": {
+ "values": []
+ }
+ }
+ },
+ {
+ "key": "by_tool",
+ "value": {
+ "arrayValue": {
+ "values": []
+ }
+ }
+ },
+ {
+ "key": "truncated",
+ "value": {
+ "boolValue": false
+ }
+ }
+ ]
+ }
+ }
+ },
+ {
+ "key": "cache",
+ "value": {
+ "kvlistValue": {
+ "values": [
+ {
+ "key": "gap_edges_s",
+ "value": {
+ "arrayValue": {
+ "values": [
+ {
+ "doubleValue": 30.0
+ },
+ {
+ "doubleValue": 60.0
+ },
+ {
+ "doubleValue": 120.0
+ },
+ {
+ "doubleValue": 300.0
+ },
+ {
+ "doubleValue": 600.0
+ },
+ {
+ "doubleValue": 1800.0
+ }
+ ]
+ }
+ }
+ },
+ {
+ "key": "hits",
+ "value": {
+ "arrayValue": {
+ "values": [
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "3"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ }
+ ]
+ }
+ }
+ },
+ {
+ "key": "misses",
+ "value": {
+ "arrayValue": {
+ "values": [
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ }
+ ]
+ }
+ }
+ },
+ {
+ "key": "write_5m",
+ "value": {
+ "intValue": "24000"
+ }
+ },
+ {
+ "key": "write_1h",
+ "value": {
+ "intValue": "8000"
+ }
+ },
+ {
+ "key": "inferred_turns",
+ "value": {
+ "intValue": "0"
+ }
+ }
+ ]
+ }
+ }
+ },
+ {
+ "key": "trajectory",
+ "value": {
+ "kvlistValue": {
+ "values": [
+ {
+ "key": "buckets",
+ "value": {
+ "intValue": "10"
+ }
+ },
+ {
+ "key": "turns",
+ "value": {
+ "arrayValue": {
+ "values": [
+ {
+ "intValue": "1"
+ },
+ {
+ "intValue": "2"
+ },
+ {
+ "intValue": "1"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ }
+ ]
+ }
+ }
+ },
+ {
+ "key": "tokens",
+ "value": {
+ "arrayValue": {
+ "values": [
+ {
+ "intValue": "120000"
+ },
+ {
+ "intValue": "240000"
+ },
+ {
+ "intValue": "120000"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ }
+ ]
+ }
+ }
+ },
+ {
+ "key": "saved",
+ "value": {
+ "arrayValue": {
+ "values": [
+ {
+ "intValue": "30000"
+ },
+ {
+ "intValue": "60000"
+ },
+ {
+ "intValue": "30000"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ },
+ {
+ "intValue": "0"
+ }
+ ]
+ }
+ }
+ },
+ {
+ "key": "kinds",
+ "value": {
+ "stringValue": "s4"
+ }
+ },
+ {
+ "key": "kinds_truncated",
+ "value": {
+ "boolValue": false
+ }
+ },
+ {
+ "key": "msgs_max",
+ "value": {
+ "intValue": "140"
+ }
+ }
+ ]
+ }
+ }
+ },
+ {
+ "key": "clients",
+ "value": {
+ "arrayValue": {
+ "values": [
+ {
+ "kvlistValue": {
+ "values": [
+ {
+ "key": "client",
+ "value": {
+ "stringValue": "claude_code"
+ }
+ },
+ {
+ "key": "n",
+ "value": {
+ "intValue": "4"
+ }
+ }
+ ]
+ }
+ }
+ ]
+ }
+ }
+ },
+ {
+ "key": "errors",
+ "value": {
+ "kvlistValue": {
+ "values": [
+ {
+ "key": "count",
+ "value": {
+ "intValue": "0"
+ }
+ },
+ {
+ "key": "by_status",
+ "value": {
+ "kvlistValue": {
+ "values": []
+ }
+ }
+ }
+ ]
+ }
+ }
+ },
+ {
+ "key": "strata",
+ "value": {
+ "arrayValue": {
+ "values": []
+ }
+ }
+ },
+ {
+ "key": "config",
+ "value": {
+ "arrayValue": {
+ "values": []
+ }
+ }
}
]
}
diff --git a/deploy/beacon/test-rollup.mjs b/deploy/beacon/test-rollup.mjs
index 3878ee2d4..8fa186e99 100644
--- a/deploy/beacon/test-rollup.mjs
+++ b/deploy/beacon/test-rollup.mjs
@@ -13,7 +13,7 @@
*/
import { readdirSync, readFileSync } from 'node:fs';
import assert from 'node:assert/strict';
-import { oldestRawDay, rollupHour } from './worker.js';
+import { inflate, oldestRawDay, rollupHour } from './worker.js';
// R2 returns at most 1000 keys per list page, so on a real hour (~4,000
// objects) the cursor loop in rollupHour is load-bearing. The stub paginates at
@@ -211,4 +211,47 @@ if (dir) {
);
}
+// 8. Upload compression. A schema-v2 event is ~8KB of repetitive JSON and
+// gzips ~6x, so fetch() sniffs the gzip magic number and inflates. Three
+// things have to hold, and none of them can be checked in production without
+// risking every upload:
+//
+// a) a real gzip body round-trips to the exact bytes that went in;
+// b) DecompressionStream exists in this runtime at all (it is the one part
+// of the ingest path with no fallback -- if it is missing, inflate throws,
+// fetch answers 400, and every compressed upload is refused);
+// c) the size cap is enforced WHILE the stream is read, not after, or a
+// decompression bomb has already been decompressed by the time it is
+// noticed.
+{
+ const gzipBytes = async (text) => {
+ const stream = new Response(new TextEncoder().encode(text)).body.pipeThrough(
+ new CompressionStream('gzip')
+ );
+ return new Uint8Array(await new Response(stream).arrayBuffer());
+ };
+
+ const payload = JSON.stringify({
+ resourceLogs: [{ scopeLogs: [{ logRecords: [{ body: { stringValue: 'x'.repeat(4000) } }] }] }],
+ });
+ const packed = await gzipBytes(payload);
+
+ assert.equal(packed[0], 0x1f, 'gzip magic byte 0');
+ assert.equal(packed[1], 0x8b, 'gzip magic byte 1');
+ assert.ok(packed.length < payload.length / 3, 'gzip did not actually compress');
+ assert.equal(await inflate(packed.buffer, 256 * 1024), payload, 'round trip changed the body');
+
+ // A bomb: highly compressible input that expands past the cap. The cap must
+ // bite, and it must bite from the stream rather than from the final size.
+ const bomb = await gzipBytes('A'.repeat(2 * 1024 * 1024));
+ assert.ok(bomb.length < 64 * 1024, 'fixture is not actually a bomb');
+ await assert.rejects(
+ () => inflate(bomb.buffer, 256 * 1024),
+ /too large/,
+ 'a decompression bomb was inflated in full'
+ );
+
+ console.log(`gzip: ${payload.length} B -> ${packed.length} B, bomb refused`);
+}
+
console.log('ok');
diff --git a/deploy/beacon/worker.js b/deploy/beacon/worker.js
index 7fea03001..43cfb29f3 100644
--- a/deploy/beacon/worker.js
+++ b/deploy/beacon/worker.js
@@ -53,6 +53,51 @@ const ALLOWED_KEYS = [
// real minimum cacheable prefix, how long a cache actually survives, and how
// far predicted cache hits are from the ones that happened.
'routing',
+
+ // ---- schema v2 -----------------------------------------------------------
+ //
+ // Additive. Every v1 key above is untouched, so a v1 client keeps storing
+ // exactly what it stores today and rows from the two versions union cleanly
+ // (`schema_version` separates them, `union_by_name` handles the rest).
+ //
+ // Read the ordering rule at the top of this list literally: this deploy must
+ // go out BEFORE the client release that starts sending these, or the first
+ // weeks of the new signals are dropped on arrival and cannot be recovered.
+ // Removing one line here is also the rollback -- it reverts that signal for
+ // every install in the wild without a client release, which is the whole
+ // reason the allowlist is server-side.
+
+ // Regret. `reread_compressed_tokens` is the counter-pressure on `saved_pct`:
+ // tokens the agent had to fetch again because compression removed them.
+ // Without it a compression ratio is unfalsifiable.
+ 'quality',
+ // Fixed-bucket distributions for quantities otherwise reported only as
+ // session sums. Ships its own bucket edges, so no reader needs to know which
+ // client version wrote a row.
+ 'hist',
+ // (content_type x strategy x yield), and the same for JSON tool output keyed
+ // by structural shape. The input term the corpus has never had.
+ // Structural descriptors only -- never TOIN's structure_hash, which is a
+ // digest of field names.
+ 'shapes',
+ // Prompt-cache survival by gap since the previous turn: the empirical TTL
+ // and minimum-prefix curve per provider.
+ 'cache',
+ // Session shape over time -- log-spaced turn buckets plus a run-length
+ // encoded turn-kind string from a closed seven-letter alphabet.
+ 'trajectory',
+ // Identified client harness, per turn. `headroom.stack` answers this by
+ // detection and resolves to a literal "proxy" for most of the fleet.
+ 'clients',
+ // 4xx, kept strictly separate from the 5xx-only `failures` above so neither
+ // metric changes meaning. 429 rate is the routing signal here.
+ 'errors',
+ // output_shaper A/B strata as validated enums. Model family is not among
+ // them -- see the note in session.py's payload().
+ 'strata',
+ // Allowlisted, slug-validated configuration, so the corpus can tell
+ // "compression underperformed" from "compression was switched off".
+ 'config',
];
// Resource attributes we keep. Same rule: allowlist, not denylist.
@@ -66,8 +111,60 @@ const ALLOWED_RESOURCE = [
'host.arch',
];
-// A beacon event is ~2KB. Anything far past that is a bug or an attack.
-const MAX_BODY_BYTES = 64 * 1024;
+// Cap on what ARRIVES, compressed or not.
+//
+// Was 64KB, chosen when the comment above it read "a beacon event is ~2KB" --
+// i.e. 32x the payload. Schema v2 made a typical event ~8KB uncompressed, and
+// a session that fills every table (the shape cross product is 10 content
+// types x 13 strategies) reaches ~82KB. That silently eroded the multiple to
+// under 1x: a busy session sending uncompressed -- the kill switch is set, or
+// the gzip fallback fired -- would have been answered 413 and, correctly, not
+// retried, so the event would simply be lost.
+//
+// Restored to ~3x the worst legitimate payload. This is not the abuse control
+// and never was: the endpoint is unauthenticated by design and the WAF rate
+// limit (60 req/min/IP, see wrangler.toml) is what bounds a bad actor. At 60
+// requests a minute the difference between these two ceilings is 3.8 MB/min
+// and 15 MB/min, neither of which is interesting to Cloudflare.
+const MAX_BODY_BYTES = 256 * 1024;
+
+// Cap on what an arriving body EXPANDS to. `raw.byteLength` cannot see this:
+// a small gzip can expand to gigabytes, and a decompression bomb that is only
+// noticed after it has been decompressed has already won. Enforced while the
+// stream is read, not after. ~3x the worst legitimate payload, same as above.
+const MAX_DECOMPRESSED_BYTES = 256 * 1024;
+
+/**
+ * Decompress a gzip body, refusing anything that expands past `limit`.
+ *
+ * Counts bytes as chunks arrive and aborts mid-stream, so a bomb costs the
+ * limit rather than whatever it would have expanded to.
+ */
+export async function inflate(buffer, limit) {
+ const stream = new Response(buffer).body.pipeThrough(new DecompressionStream('gzip'));
+ const reader = stream.getReader();
+ const chunks = [];
+ let total = 0;
+ try {
+ for (;;) {
+ const { done, value } = await reader.read();
+ if (done) break;
+ total += value.byteLength;
+ if (total > limit) throw new Error('decompressed body too large');
+ chunks.push(value);
+ }
+ } finally {
+ // Releases the stream on the bomb path; a no-op once it has closed.
+ reader.cancel().catch(() => {});
+ }
+ const joined = new Uint8Array(total);
+ let offset = 0;
+ for (const chunk of chunks) {
+ joined.set(chunk, offset);
+ offset += chunk.byteLength;
+ }
+ return new TextDecoder().decode(joined);
+}
/** OTLP AnyValue -> plain JS. The inverse of _any_value() in session.py. */
function unwrap(value) {
@@ -315,9 +412,31 @@ export default {
let records;
try {
- records = extract(JSON.parse(new TextDecoder().decode(raw)));
+ // Sniff the gzip magic number rather than trusting Content-Encoding.
+ // Three things can each independently decide whether a body arrives
+ // compressed -- the client, Cloudflare's edge, and any proxy between
+ // them -- and only the bytes know which of them acted. Keying off the
+ // header instead means a body that something already decompressed gets
+ // fed to DecompressionStream, or vice versa, and every upload 400s.
+ //
+ // It also means an old client that never sets the header keeps working
+ // through the identical code path it uses today: no header, no magic
+ // number, straight to TextDecoder.
+ const bytes = new Uint8Array(raw);
+ const gzipped = bytes.length >= 2 && bytes[0] === 0x1f && bytes[1] === 0x8b;
+ const text = gzipped
+ ? await inflate(raw, MAX_DECOMPRESSED_BYTES)
+ : new TextDecoder().decode(raw);
+ records = extract(JSON.parse(text));
} catch {
// Malformed input is not worth a retry storm from clients.
+ //
+ // This 400 is also load-bearing in the other direction: it is what a
+ // client that gzipped against a Worker too old to inflate sees, and
+ // _post_blocking treats a 4xx on a compressed body as "this endpoint
+ // does not do gzip", falls back to sending it uncompressed, and stops
+ // compressing for the life of the process. So deploying the client
+ // before this Worker costs one retry per process, not the data.
return new Response('bad request', { status: 400 });
}
if (records.length === 0) return new Response(null, { status: 204 });
diff --git a/headroom/cli/main.py b/headroom/cli/main.py
index 74fdaad8a..e2b8755e6 100644
--- a/headroom/cli/main.py
+++ b/headroom/cli/main.py
@@ -78,6 +78,7 @@ def _register_commands() -> None:
recover, # noqa: F401
rollout, # noqa: F401
savings, # noqa: F401
+ telemetry, # noqa: F401
tools, # noqa: F401
update, # noqa: F401
wrap, # noqa: F401
diff --git a/headroom/cli/telemetry.py b/headroom/cli/telemetry.py
new file mode 100644
index 000000000..e68e85aad
--- /dev/null
+++ b/headroom/cli/telemetry.py
@@ -0,0 +1,147 @@
+"""``headroom telemetry`` — show exactly what the beacon would send.
+
+The beacon is opt-OUT and on by default. "The receiver is open source, go read
+worker.js" is a good answer to what happens to the data, but it is a poor
+answer to what LEAVES YOUR MACHINE — that requires reading two files, trusting
+that the copy on GitHub matches the copy installed, and reconstructing a
+payload by hand.
+
+This prints the real thing: the current process's resource attributes and a
+session payload built by the same `_Session.payload()` the beacon uses, with
+representative counters folded in. Nothing here is a mock-up of the schema; if
+a field can appear on the wire, it appears here.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+import click
+
+from .main import main
+
+
+def _sample_payload() -> dict[str, Any]:
+ """One session payload, built through the real aggregator.
+
+ Uses the actual fold path rather than hand-writing a dict, so this cannot
+ drift from what is sent. The token counts are invented; the SHAPE — every
+ key, and the fact that every value is a counter, a ratio or a bounded slug
+ — is exactly what a real session produces.
+ """
+ from headroom.telemetry import session as telemetry_session
+
+ class _Example:
+ provider = "anthropic"
+ model = "gpt-4o"
+ original_tokens = 120_000
+ attempted_input_tokens = 40_000
+ optimized_tokens = 90_000
+ provider_input_tokens = 90_000
+ output_tokens = 1_200
+ tokens_saved = 30_000
+ cache_read_tokens = 60_000
+ cache_write_tokens = 8_000
+ cache_write_5m_tokens = 6_000
+ cache_write_1h_tokens = 2_000
+ uncached_input_tokens = 22_000
+ status_code = 200
+ total_latency_ms = 4_200.0
+ overhead_ms = 48.0
+ ttfb_ms = 900.0
+ num_messages = 140
+ client = "claude_code"
+ transforms_applied = ("crush",)
+ tags: dict[str, Any] = {}
+ waste_signals = {"reread": 900, "reread_compressed": 400, "json_bloat": 50}
+
+ rows: list[dict[str, Any]] = []
+ aggregator = telemetry_session.SessionAggregator(rows.append)
+ for index in range(3):
+ aggregator.record(_Example(), now=1_000.0 + index * 45.0)
+ aggregator.flush_all()
+ return rows[-1]
+
+
+ "--show",
+ "show",
+ is_flag=True,
+ help="Print the exact payload the beacon would send, as JSON.",
+)
+ "--json",
+ "as_json",
+ is_flag=True,
+ help="Print JSON only, with no explanatory footer. Implies --show.",
+)
+def telemetry(show: bool, as_json: bool) -> None:
+ """Show what Headroom's telemetry beacon collects, and whether it is on.
+
+ \b
+ Examples:
+ headroom telemetry Current state and the two switches
+ headroom telemetry --show The exact payload, field by field
+ """
+ from headroom.telemetry.beacon import is_beacon_enabled, is_telemetry_enabled
+ from headroom.telemetry.session import (
+ DEFAULT_ENDPOINT,
+ SCHEMA_VERSION,
+ _gzip_enabled,
+ read_install_id,
+ resource_attributes,
+ )
+
+ beacon_on = is_beacon_enabled()
+ local_on = is_telemetry_enabled()
+ known_id = read_install_id()
+ # Reported because it changes what goes on the wire, and because while the
+ # transport is staged this is the fastest way to answer "was that upload
+ # compressed?" without instrumenting anything.
+ gzip_on = _gzip_enabled()
+
+ if show or as_json:
+ report = {
+ "beacon_enabled": beacon_on,
+ "schema_version": SCHEMA_VERSION,
+ "endpoint": DEFAULT_ENDPOINT,
+ "gzip": gzip_on,
+ "install_id": known_id,
+ # create_install_id=False, and it is load-bearing: the default
+ # mints and persists an id, so building this report would be the
+ # act that identifies the machine. Filtering the id out of the
+ # OUTPUT is not enough -- by then the file exists.
+ "resource": resource_attributes(create_install_id=False),
+ "session_event": _sample_payload(),
+ }
+ click.echo(json.dumps(report, indent=2, sort_keys=True))
+ if as_json:
+ return
+ click.echo()
+ click.echo(
+ "The COUNTS above are made up. The FIELDS are not: this payload is built\n"
+ "by the same code the beacon uses, so every key that can ever be sent is\n"
+ "shown, and no other key can be.\n"
+ "\n"
+ "Every value is a counter, a ratio, or a slug from a fixed list in the\n"
+ "source. No prompts, code, file paths, tool names, project names or error\n"
+ "text can appear — see headroom/telemetry/session.py for what is built,\n"
+ "and deploy/beacon/worker.js for the allowlist it is filtered through on\n"
+ "arrival. Your install_id is a random UUID, not a machine fingerprint;\n"
+ "delete ~/.headroom/install_id to reset it. Your IP is never read."
+ )
+ return
+
+ click.echo(f"Upload beacon: {'ON (opt-out)' if beacon_on else 'OFF'}")
+ click.echo(f"Local stats: {'ON' if local_on else 'OFF (opt-in)'}")
+ click.echo(f"Endpoint: {DEFAULT_ENDPOINT}")
+ click.echo(
+ f"Compression: {'gzip' if gzip_on else 'off (plain JSON)'}"
+ f"{'' if gzip_on else ' — enable: HEADROOM_BEACON_GZIP=1'}"
+ )
+ click.echo(f"Install ID: {known_id or '(none yet — created on first upload)'}")
+ click.echo()
+ click.echo("Turn the beacon off: HEADROOM_BEACON=off (or DO_NOT_TRACK=1)")
+ click.echo("See the exact payload: headroom telemetry --show")
diff --git a/headroom/telemetry/beacon.py b/headroom/telemetry/beacon.py
index eda40869e..72556e761 100644
--- a/headroom/telemetry/beacon.py
+++ b/headroom/telemetry/beacon.py
@@ -124,17 +124,24 @@ def format_telemetry_notice(*, prefix: str = "") -> str:
# so this notice is the only place a user learns it is running at all —
# surprise is what turns anonymous telemetry into a trust incident.
#
- # Wording: lead with what the data *is* (compression ratios) and why it is
- # useful (understanding when compression works), not with the fact of
+ # Wording: lead with what the data *is* (counters) and why it is useful
+ # (understanding when compression works), not with the fact of
# transmission. "Stats are sent to Headroom Labs" is accurate but reads like
# extraction, which misrepresents a payload that is entirely counters. The
# reassurance has to stay concrete, though — "never prompts, code, or file
# paths" names the things people actually worry about, and every one of
# those is enforced by the allowlist, not just promised here.
+ #
+ # "compression stats" until schema v2, which widened the payload to cache
+ # behaviour, session shape, harness and configuration. All still counters,
+ # all still content-free — but this notice is the only place an opt-out
+ # beacon is disclosed at all, so it has to describe what is actually sent
+ # rather than the narrowest true thing. `headroom telemetry --show` prints
+ # the exact payload for anyone who wants the specifics.
if beacon:
return (
- f"{prefix}Telemetry: anonymous compression stats — never prompts, code, or "
- "file paths. Helps us improve compression | Off: HEADROOM_BEACON=off"
+ f"{prefix}Telemetry: anonymous usage counters — never prompts, code, or file "
+ "paths. See: headroom telemetry --show | Off: HEADROOM_BEACON=off"
)
return (
f"{prefix}Telemetry: ENABLED (local aggregate stats only — nothing sent externally) | "
diff --git a/headroom/telemetry/session.py b/headroom/telemetry/session.py
index fdd9fd433..b4bd6b87f 100644
--- a/headroom/telemetry/session.py
+++ b/headroom/telemetry/session.py
@@ -42,11 +42,66 @@ wire format is a stable spec and plain stdlib gets us there.
The record body is an OTLP kvlist (not a JSON string) so the collector's
``keep_keys(body, [...])`` allowlist can introspect and drop unknown fields
server-side. That is the only control point for clients already in the wild.
+
+Bodies can be gzipped (``Content-Encoding: gzip``, as OTLP/HTTP specifies). The
+kvlist encoding costs about 3.8x over plain JSON -- every integer becomes
+``{"intValue":"1200"}`` -- and the result is repetitive enough to compress ~6x,
+which is worth several times more than every available schema trim put
+together. It also restores the premise the cumulative-snapshot design rests on:
+``payload`` justifies re-sending everything on each heartbeat because "the
+redundancy is ~2KB per report, which is free", and at schema v2 that report is
+~8KB uncompressed and ~1.3KB gzipped. Compression can turn itself off -- see
+``_post_blocking`` -- so an endpoint that cannot read it costs a retry, not the
+data.
+
+It is nonetheless OFF in this release: v2 and the transport are separate
+mechanisms and ship in separate releases, so that a failure in the first is
+never ambiguous. See ``_GZIP_DEFAULT``. Note that it saves upload bandwidth
+only -- the receiver stores plain NDJSON either way, so corpus size and read
+times are identical with it on or off.
+
+# Schema versions
+
+``schema_version: 1`` reported volume: how many tokens moved, which strategies
+ran, and how much was saved. It answered "how much did we save?" and nothing
+else, because every counter in it is a session-wide SUM and none of them
+describes the input.
+
+``schema_version: 2`` is strictly additive — every v1 field keeps its name, its
+type and its arithmetic, and the folding for the new signals is isolated in
+:func:`_fold_extra` so a bug there cannot reach one. It adds:
+
+* ``quality`` — the regret label. ``reread_compressed_tokens`` is tokens the
+ agent had to fetch again *because compression removed them*, which is the
+ only thing that makes ``saved_pct`` falsifiable: without it, deleting the
+ entire context scores 100%.
+* ``hist`` — fixed-bucket distributions for quantities previously reported only
+ as totals, since a sum cannot distinguish uniformly-mediocre from bimodal.
+ Each ships its own bucket edges, so a row is readable without knowing which
+ client version wrote it.
+* ``shapes`` — ``(content_type -> strategy -> yield)``, the input term the
+ corpus never had. ``by_strategy`` can rank compressors against each other but
+ cannot say which one suits a given piece of content, which is the decision
+ the router actually makes.
+* ``cache`` — prompt-cache hit rate indexed by the gap since the previous turn,
+ i.e. an empirical survival curve for a TTL no provider documents accurately.
+* ``trajectory`` — the session's shape over time rather than its totals.
+* ``clients`` / ``errors`` / ``strata`` / ``config`` — segmentation. 4xx is
+ counted in ``errors`` and deliberately NOT folded into ``failures``, which
+ has been 5xx-only in every row of the corpus and must stay that way.
+
+Everything added is a counter, a ratio, or a slug from a vocabulary that is
+closed in code. Two things are still refused on principle: anything derived
+from prompt or file content (which is why ``turn_id`` and TOIN's
+``structure_hash`` are absent — both are digests of user data), and the source
+IP, which the receiver never reads. ``headroom telemetry --show`` prints the
+exact payload, so none of this has to be taken on trust.
"""
from __future__ import annotations
import atexit
+import gzip
import json
import logging
import os
@@ -55,9 +110,10 @@ import re
import secrets
import threading
import time
+import urllib.error
import urllib.request
import uuid
-from collections.abc import Callable
+from collections.abc import Callable, Iterable
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
@@ -73,7 +129,7 @@ logger = logging.getLogger(__name__)
# package. Move this to otlp.headroomlabs.ai (one CNAME, or a Cloudflare zone)
# before cutting a release that contains it.
DEFAULT_ENDPOINT = "https://headroom-beacon.headroom-beacon.workers.dev/v1/logs"
-SCHEMA_VERSION = 1
+SCHEMA_VERSION = 2
# A gap longer than this closes the session. Agent loops are bursty; 15 minutes
# separates "reading the diff before the next prompt" from "walked away".
@@ -85,6 +141,91 @@ FLUSH_INTERVAL_S = 300.0
_POST_TIMEOUT_S = 5.0
+# Level 6 is zlib's default and the knee of the curve here: a real payload is
+# ~8KB of repetitive JSON and compresses ~6x, and level 9 buys under 2% more for
+# noticeably more CPU. This runs on a daemon thread, never the request path.
+_GZIP_LEVEL = 6
+
+# The only statuses that mean "I could not read this body". Narrow on purpose.
+#
+# 413 in particular must NOT be here. It means the body was too big, and the
+# fallback answers a refusal by re-sending the SAME payload uncompressed, which
+# is ~6x larger -- so a 413 would trigger a retry guaranteed to 413 again, and
+# permanently disable the compression that was the only thing keeping the
+# request under the cap. The rest of the 4xx range (401, 403, 404, 429) is
+# equally not about encoding, and an uncompressed retry does not help there
+# either.
+_GZIP_REFUSED_STATUSES = frozenset((400, 415))
+
+# Compression ships OFF and is opted into with HEADROOM_BEACON_GZIP=1.
+#
+# Staged deliberately. Schema v2 and the gzip transport are two independent new
+# mechanisms, and shipping both in one release means any upload failure needs
+# disambiguating before it can be fixed. With this default the first v2 release
+# sends plain bodies through the byte-identical path v1 already uses, so a
+# failure there is unambiguously the payload.
+#
+# It also keeps `inflate()` in the Worker dead: the receiver sniffs gzip magic
+# bytes, so if nothing compresses, nothing reaches the one code path with no
+# fallback. Flip this to True in a later release once v2 itself is proven.
+#
+# Note what this default is NOT: a rollback. A released client reads the
+# operator's environment, not ours, so this cannot be changed for installs in
+# the wild. The retroactive switch is server-side -- have the Worker answer a
+# gzipped body with 415 and every client falls back to plain and stays there
+# for the life of the process. Same reason the privacy allowlist lives there.
+_GZIP_DEFAULT = False
+
+# Set once the far end proves it cannot take a compressed body -- see
+# `_post_blocking`. Process-wide rather than per-endpoint: one endpoint per
+# process.
+_gzip_lock = threading.Lock()
+_gzip_supported = True
+
+
+class _CompressionRejected(Exception):
+ """The endpoint 4xx'd a gzipped body. Retry it uncompressed."""
+
+
+def _gzip_enabled() -> bool:
+ """Whether to compress the next upload.
+
+ Off unless the operator opts in with HEADROOM_BEACON_GZIP=1, and off
+ regardless once this process has watched the endpoint reject a compressed
+ body. Both an explicit off-value and an unrecognised value mean off, so a
+ typo degrades to the safe, already-proven transport rather than to gzip.
+
+ Cannot raise. It is called from `_post_blocking` before that function's
+ try block, and `_post_blocking` runs as a bare daemon-thread target and as
+ an atexit handler -- both places where an escaping exception is printed to
+ the user's terminal. Telemetry is never allowed to do that, so a failure
+ here degrades to "do not compress" instead.
+ """
+ try:
+ from headroom.telemetry.beacon import _OFF_VALUES, _ON_VALUES
+
+ raw = os.environ.get("HEADROOM_BEACON_GZIP", "").lower().strip()
+ if raw in _ON_VALUES:
+ enabled = True
+ elif raw in _OFF_VALUES:
+ enabled = False
+ else:
+ enabled = _GZIP_DEFAULT
+ if not enabled:
+ return False
+ with _gzip_lock:
+ return _gzip_supported
+ except Exception:
+ logger.debug("telemetry: gzip gate failed; sending uncompressed", exc_info=True)
+ return False
+
+
+def _disable_gzip() -> None:
+ global _gzip_supported
+ with _gzip_lock:
+ _gzip_supported = False
+
+
# Reason/enum values are bounded vocabularies in code, but they reach us as
# free strings. Validate rather than trust: anything off-pattern becomes
# "other" so a future tag value cannot smuggle text onto the wire.
@@ -100,6 +241,136 @@ _REASON_TAGS = ("passthrough_reason", "image_skip_reason", "memory_skip_reason")
# Matches the same guard on `requests_by_stack` (MAX_DISTINCT_STACKS).
MAX_STRATEGIES = 32
+# ---------------------------------------------------------------- schema v2 --
+#
+# Everything below is additive. Not one v1 counter changes meaning, and the
+# folding for all of it lives in `_fold_extra` rather than `_fold`, so a bug in
+# a new signal cannot corrupt a number the corpus already depends on — the
+# worst case is a missing v2 key. See the note on `_fold_extra`.
+
+# Fixed histogram edges, shipped alongside the counts so a reader never has to
+# know which client version wrote a row. `counts` is always len(edges) + 1: one
+# bucket per interval plus the open-ended tail.
+#
+# Sums answer "how much"; only a distribution answers "how consistently". A
+# session at saved_pct 12 is either uniformly mediocre or half-brilliant and
+# half-useless, and those need opposite fixes.
+_HIST_EDGES: dict[str, tuple[float, ...]] = {
+ # Per-turn request size, pre-compression.
+ "turn_tokens": (1_000, 4_000, 16_000, 64_000, 128_000, 256_000),
+ # Per-turn saved/original, in percent. Only defined when original > 0.
+ "saved_pct": (0.5, 5.0, 10.0, 20.0, 40.0, 60.0),
+ # Per-turn compression dispatch time.
+ "overhead_ms": (1.0, 5.0, 25.0, 100.0, 500.0, 2_000.0),
+ # Time to first upstream byte. 0 means unmeasured, not fast, so zeros are
+ # excluded rather than piled into the first bucket.
+ "ttfb_ms": (200.0, 500.0, 1_000.0, 2_500.0, 5_000.0, 15_000.0),
+ # Messages in the request. Grows monotonically within an agent loop, so the
+ # shape of this is the shape of the conversation.
+ "msgs": (5, 20, 50, 100, 250, 500),
+}
+
+# Gap since the previous turn, in seconds, for the cache-survival curve. The
+# provider's real prompt-cache TTL is an empirical question — nobody documents
+# it accurately and it drifts — and (gap -> hit rate) across a fleet is the
+# measurement. Edges bracket the two published Anthropic TTLs (5m, 1h).
+_CACHE_GAP_EDGES: tuple[float, ...] = (30.0, 60.0, 120.0, 300.0, 600.0, 1_800.0)
+
+# Turn-index buckets for the trajectory curve: turn 1, turns 2-3, 4-7, ... 512+.
+# Log-spaced rather than deciles because a session's turn count is not known
+# until it ends, and a cumulative snapshot cannot re-bin what it already wrote.
+# Ten buckets covers a 1000-turn session, which is past the observed maximum.
+_TRAJECTORY_BUCKETS = 10
+
+# Turn kinds, in priority order — the first that matches wins. A closed
+# alphabet: `_turn_kind` can only ever return one of these, so no free text
+# reaches the run-length string.
+#
+# c served from Headroom's own response cache (provider never called)
+# f upstream 5xx
+# e upstream 4xx (429 lives here, and is the interesting one)
+# p passthrough — compression was bypassed
+# z compression ran and removed nothing
+# s compression ran and removed something
+# o none of the above (nothing was eligible, no bypass reason given)
+_TURN_KINDS = frozenset("cfepzso")
+
+# Cap on the run-length trajectory string. A 1400-turn session that alternates
+# kinds every turn would otherwise write ~2.8KB of runs on its own.
+MAX_TRAJECTORY_RUNS = 64
+
+# Cardinality cap for the v2 maps that take a slug from a bounded vocabulary
+# but are still fed by a free string somewhere upstream. Same reasoning and the
+# same number as MAX_STRATEGIES.
+MAX_SHAPES = 64
+
+# Row cap for the two shape TABLES, which is a different question: their
+# vocabulary is a cross product, not a list. ContentType has 9 members and
+# CompressionStrategy 12, plus the "other" each collapses to, so
+# `shapes.by_content` can legitimately hold 10 x 13 = 130 distinct rows. Capped
+# at MAX_SHAPES the table would drop 66 of them — and drop them
+# FIRST-COME-WINS, which does not merely lose rows, it biases the table toward
+# whatever the session happened to route early. That table is the one a routing
+# policy would be fitted on, so a systematic bias in it is worse than its
+# bytes. Sized above the cross product so the cap cannot bite in normal
+# operation and remains purely a guard against a caller inventing keys.
+MAX_SHAPE_ROWS = 160
+
+# The only waste signals that reach the wire, by name. `waste_signals` is typed
+# `dict[str, int]` and has more than one producer, so an allowlist here — not a
+# copy of whatever the dict happens to hold — is what keeps a future key from
+# shipping before anyone has looked at it.
+#
+# `reread_compressed` is the reason this section exists. Headroom computes it
+# already (see WasteSignals in config.py): re-reads attributable to our own
+# over-compression rather than to agent behaviour. That is a regret label for
+# the compressor, and a compression ratio without one is unfalsifiable — you
+# can drive `saved_pct` to 90 by deleting the context and nothing here would
+# object.
+_WASTE_KEYS = (
+ "json_bloat",
+ "html_noise",
+ "base64",
+ "whitespace",
+ "dynamic_date",
+ "repetition",
+)
+
+# The output_shaper A/B labels, duplicated from
+# `headroom.proxy.output_savings_policy` rather than imported. The telemetry
+# package deliberately does not import `headroom.proxy` (see `record_stack`),
+# and two string constants are a smaller price than that dependency.
+_STRATUM_TREATMENT = "output_shaper:stratum:"
+_STRATUM_CONTROL = "output_shaper:control:"
+
+# Environment variables whose values may be reported, by name. An allowlist,
+# never a dump of the environment: a variable absent from this tuple cannot
+# reach the wire whatever it holds, and every value that does is put through
+# `_safe_slug` first, so a var that unexpectedly holds a URL or a path reports
+# "other" rather than its contents.
+#
+# Endpoints, tokens, paths, ports and hostnames are excluded by construction —
+# only settings whose vocabulary is a small closed set are listed. This answers
+# the question the corpus currently cannot: whether a session with poor savings
+# is a compression failure or simply a configuration.
+_CONFIG_ENV_KEYS = (
+ "HEADROOM_MODE",
+ "HEADROOM_SAVINGS_PROFILE",
+ "HEADROOM_DEPLOYMENT_PROFILE",
+ "HEADROOM_ROLLOUT_CHANNEL",
+ "HEADROOM_CCR_BACKEND",
+ "HEADROOM_MEMORY_INJECTION_MODE",
+ "HEADROOM_OUTPUT_SHAPER",
+ "HEADROOM_OUTPUT_HOLDOUT",
+ "HEADROOM_LOSSLESS",
+ "HEADROOM_DEDUPE",
+ "HEADROOM_STATELESS",
+ "HEADROOM_CODE_AWARE_ENABLED",
+ "HEADROOM_PROTECT_TOOL_RESULTS",
+ "HEADROOM_INTERCEPT_TOOL_RESULTS",
+ "HEADROOM_DISABLE_KOMPRESS",
+)
+
# Compression events arrive on the compression executor thread, mid-request,
# before that request's outcome ever reaches `SessionAggregator.record`. They
# are staged here rather than written straight into the live session, which
@@ -126,6 +397,17 @@ _staged_strategies: dict[str, list[int]] = {}
# and the same reason: the proxy sees the X-Headroom-Stack header per request,
# and this is the only place the beacon can learn it without importing proxy.
_staged_stacks: dict[str, int] = {}
+# (content_type, strategy) -> [sections, tokens_in, tokens_out]. Staged for the
+# same reason as `_staged_strategies`: ContentRouter observes on the compression
+# executor thread, mid-request, and must not take the aggregator lock.
+#
+# `_staged_strategies` records what each compressor YIELDED but never what it
+# was HANDED, so the corpus has no left-hand side and cannot express the one
+# function worth learning: shape -> strategy -> yield. This is that term.
+_staged_shapes: dict[tuple[str, str], list[int]] = {}
+# tool-output shape slug -> [events, tokens_in, tokens_out]. The structural
+# descriptor only — never TOIN's `structure_hash`. See `record_tool_shape`.
+_staged_tool_shapes: dict[str, list[int]] = {}
def _pct(numerator: float, denominator: float) -> float:
@@ -145,6 +427,87 @@ def _safe_slug(value: Any) -> str:
return text if _SLUG_RE.match(text) else "other"
+def _client_slug(value: Any) -> str:
+ """Slug a client-harness id, translating `-` to `_` first.
+
+ `classify_client` returns hyphenated ids -- `claude-code`, `claude-vscode`,
+ `anthropic-cli` -- and `_SLUG_RE` does not admit a hyphen, so slugging one
+ directly reports "other" for three of the thirteen known harnesses,
+ including the one that drives most of the fleet. That defeats the entire
+ point of reporting the field.
+
+ Deliberately NOT fixed by widening `_SLUG_RE`: that validator guards the
+ v1 skip reasons and strategy names, where a value that fails today already
+ ships as "other" and has done so in every row of the corpus. Widening it
+ would quietly change what those existing fields mean. A separate function
+ for the one field with a different convention is the smaller change.
+ """
+ text = str(value or "").strip().lower().replace("-", "_")
+ return text if _SLUG_RE.match(text) else "other"
+
+
+def _bucket(edges: tuple[float, ...], value: float) -> int:
+ """Index of the first edge `value` falls under, else the open-ended tail."""
+ for index, edge in enumerate(edges):
+ if value < edge:
+ return index
+ return len(edges)
+
+
+def _bump(counter: dict[str, int], key: str, *, cap: int = MAX_SHAPES) -> None:
+ """Increment a bounded slug->count map. Over the cap, new keys are dropped.
+
+ Dropped rather than bucketed into "other": every caller here feeds a
+ vocabulary that is closed in code, so hitting the cap means something
+ upstream is generating keys per request, and a silently growing "other"
+ would hide that where a flat count makes it obvious.
+ """
+ if key not in counter and len(counter) >= cap:
+ return
+ counter[key] = counter.get(key, 0) + 1
+
+
+def _turn_kind(*, cached: bool, status: int, passthrough: bool, attempted: int, saved: int) -> str:
+ """Classify one turn into the closed `_TURN_KINDS` alphabet.
+
+ Structural only: every input is a status code, a token count, or a flag the
+ proxy already set. Nothing here reads content, and the return value is one
+ of seven literals, so the trajectory string cannot carry free text.
+ """
+ if cached:
+ return "c"
+ if status >= 500:
+ return "f"
+ if 400 <= status < 500:
+ return "e"
+ if passthrough:
+ return "p"
+ if attempted > 0:
+ return "s" if saved > 0 else "z"
+ return "o"
+
+
+def _config_snapshot() -> dict[str, str]:
+ """Allowlisted, slug-validated view of the operator's configuration.
+
+ Two layers of protection, because one is not enough for anything that reads
+ the environment: only the names in `_CONFIG_ENV_KEYS` are consulted at all,
+ and every value that survives that is put through `_safe_slug`, so a
+ variable that unexpectedly holds a path, a URL or a token reports "other".
+
+ Unset variables are omitted rather than reported as a default, so absent
+ means "operator expressed no preference" and the reader is never guessing
+ which release's default applied.
+ """
+ out: dict[str, str] = {}
+ for name in _CONFIG_ENV_KEYS:
+ raw = os.environ.get(name)
+ if raw is None or not raw.strip():
+ continue
+ out[name.removeprefix("HEADROOM_").lower()] = _safe_slug(raw)
+ return out
+
+
_model_cache: dict[str, str | None] = {}
@@ -199,25 +562,49 @@ def _install_id_path() -> Path:
return config_dir() / "install_id"
+def read_install_id() -> str | None:
+ """The install id if one already exists, WITHOUT creating one.
+
+ Split out of `install_id` because minting an identifier is a side effect,
+ and there are callers that must not have it. `headroom telemetry --show`
+ is the one that matters: someone running it to find out what leaves their
+ machine — very plausibly on the way to switching the beacon off — must not
+ have that inspection be the thing that gives the machine its id.
+ """
+ global _install_id
+ with _identity_lock:
+ if _install_id is not None:
+ return _install_id
+ try:
+ existing = _install_id_path().read_text().strip()
+ except OSError:
+ return None
+ if existing:
+ _install_id = existing
+ return _install_id
+ return None
+
+
def install_id() -> str:
"""Random per-install id. Not a machine fingerprint — delete the file to reset.
Deliberately not derived from hostname, MAC, or any hardware property:
anything a user cannot change reads as tracking, and a random UUID answers
every question the beacon actually asks.
+
+ Creates and persists one if none exists. Use :func:`read_install_id` where
+ that side effect is not wanted.
"""
global _install_id
+ existing = read_install_id()
+ if existing is not None:
+ return existing
with _identity_lock:
+ # Re-check: another thread may have won the race while this one was
+ # outside the lock in read_install_id().
if _install_id is not None:
return _install_id
path = _install_id_path()
- try:
- existing = path.read_text().strip()
- if existing:
- _install_id = existing
- return _install_id
- except OSError:
- pass
_install_id = uuid.uuid4().hex
try:
path.parent.mkdir(parents=True, exist_ok=True)
@@ -230,7 +617,10 @@ def install_id() -> str:
def resource_attributes(
- *, install_mode: str | None = None, stack: str | None = None
+ *,
+ install_mode: str | None = None,
+ stack: str | None = None,
+ create_install_id: bool = True,
) -> dict[str, str]:
"""Set once per process; every signal inherits these for free.
@@ -245,10 +635,16 @@ def resource_attributes(
attrs = {
"service.name": "headroom",
"service.version": get_version(),
- "headroom.install_id": install_id(),
"os.type": platform.system().lower(),
"host.arch": platform.machine().lower(),
}
+ # `create_install_id=False` is for callers that only want to LOOK at what
+ # would be sent. Minting an id is a side effect, and an inspection command
+ # must not have it — see `read_install_id`. The key is omitted rather than
+ # blank so a reader can tell "no id yet" from "id withheld".
+ resolved = install_id() if create_install_id else read_install_id()
+ if resolved:
+ attrs["headroom.install_id"] = resolved
if install_mode:
attrs["headroom.install_mode"] = install_mode
# Detect when the caller did not supply one. Every caller so far supplies
@@ -316,6 +712,79 @@ class _Session:
providers: set[str] = field(default_factory=set)
models: set[str] = field(default_factory=set)
+ # ---------------------------------------------------------- schema v2 --
+ # Additive only. Every field below is a running total like the v1 counters
+ # above it, so the "highest seq per (install, session) is the whole
+ # session" dedupe rule is unchanged and a dropped heartbeat still costs
+ # nothing. Anything whose natural form is a last-value (msgs_max) is
+ # written as a maximum instead, which behaves identically under that rule.
+ hists: dict[str, list[int]] = field(
+ default_factory=lambda: {
+ name: [0] * (len(edges) + 1) for name, edges in _HIST_EDGES.items()
+ }
+ )
+ waste: dict[str, int] = field(default_factory=dict)
+ waste_turns: int = 0
+ reread_tokens: int = 0
+ reread_compressed_tokens: int = 0
+ shapes: dict[tuple[str, str], list[int]] = field(default_factory=dict)
+ tool_shapes: dict[str, list[int]] = field(default_factory=dict)
+ cache_gap_hits: list[int] = field(default_factory=lambda: [0] * (len(_CACHE_GAP_EDGES) + 1))
+ cache_gap_misses: list[int] = field(default_factory=lambda: [0] * (len(_CACHE_GAP_EDGES) + 1))
+ cache_write_5m: int = 0
+ cache_write_1h: int = 0
+ cache_inferred_turns: int = 0
+ traj_turns: list[int] = field(default_factory=lambda: [0] * _TRAJECTORY_BUCKETS)
+ traj_tokens: list[int] = field(default_factory=lambda: [0] * _TRAJECTORY_BUCKETS)
+ traj_saved: list[int] = field(default_factory=lambda: [0] * _TRAJECTORY_BUCKETS)
+ # [kind, run_length] pairs, capped at MAX_TRAJECTORY_RUNS.
+ kind_runs: list[list[Any]] = field(default_factory=list)
+ kinds_truncated: bool = False
+ msgs_max: int = 0
+ clients: dict[str, int] = field(default_factory=dict)
+ # 4xx is counted HERE and never in `failures` / `failure_statuses`. Those
+ # two are 5xx-only in every row of the existing corpus, and widening them
+ # would silently redefine a metric rather than add one — a session that
+ # rate-limited twice would start reading as a session that failed twice.
+ client_errors: int = 0
+ client_error_statuses: dict[str, int] = field(default_factory=dict)
+ strata_arm: dict[str, int] = field(default_factory=dict)
+ strata_turn_kind: dict[str, int] = field(default_factory=dict)
+ strata_input_bucket: dict[str, int] = field(default_factory=dict)
+ strata_tools: dict[str, int] = field(default_factory=dict)
+
+ def observe(self, name: str, value: float) -> None:
+ """Record one sample into a named histogram."""
+ counts = self.hists.get(name)
+ if counts is not None:
+ counts[_bucket(_HIST_EDGES[name], value)] += 1
+
+ def append_kind(self, kind: str) -> None:
+ """Extend the run-length trajectory string by one turn.
+
+ Stops dead at the cap rather than continuing to grow the last run.
+ Letting a matching kind keep incrementing looks harmless and is not:
+ once a run has been dropped the remaining turns are no longer
+ contiguous, so `s6` would claim six consecutive turns of `s` where the
+ session actually did `s q s q s q ...`. A run length that is really a
+ filtered count is worse than a short string, because nothing in the
+ payload says which one you are reading.
+
+ Stopping keeps the string a faithful PREFIX of the session — every run
+ in it is true — with `kinds_truncated` marking that more happened after
+ it. `trajectory.turns` is uncapped and still sums to `session.turns`,
+ so the turn count is never lost.
+ """
+ if self.kinds_truncated:
+ return
+ if self.kind_runs and self.kind_runs[-1][0] == kind:
+ self.kind_runs[-1][1] += 1
+ return
+ if len(self.kind_runs) >= MAX_TRAJECTORY_RUNS:
+ self.kinds_truncated = True
+ return
+ self.kind_runs.append([kind, 1])
+
def payload(self, reason: str) -> dict[str, Any]:
"""Cumulative snapshot of this session. Content-free.
@@ -333,7 +802,12 @@ class _Session:
Cumulative also makes the stream loss-tolerant: a dropped report costs
nothing, because the next one restates everything. Deltas would leave a
- permanent hole. The redundancy is ~2KB per report, which is free.
+ permanent hole. The redundancy is ~1.3KB per report on the wire, which
+ is free -- but only because the body is gzipped. Uncompressed, a
+ schema-v2 report is ~8KB and re-sending it every five minutes is not
+ free at all; see the module docstring. Anything that grows the payload
+ further should be weighed against this line, not against the ~2KB it
+ used to say.
``seq`` increments on every emission, so this is not a pure read.
"""
@@ -490,9 +964,211 @@ class _Session:
# (500/502/503/504/529), so this needs no slug bounding.
"failure_statuses": dict(self.failure_statuses),
}
+ # Merged rather than built inline, so the isolation the v2 fields claim
+ # elsewhere is structural here too. Built inline, one bad expression in
+ # any v2 section aborts the whole dict literal and the event is dropped
+ # -- taking `tokens.saved` and every other v1 counter with it. The worst
+ # case has to be a missing v2 key, never a lost session.
+ try:
+ snapshot.update(self._payload_v2())
+ except Exception:
+ logger.debug("telemetry: v2 payload section failed", exc_info=True)
self.seq += 1
return snapshot
+ def _payload_v2(self) -> dict[str, Any]:
+ """The schema-v2 half of the snapshot. See `payload`."""
+ return {
+ # Nothing below reads or rewrites a key above it. Dropping any one
+ # of these from the worker's ALLOWED_KEYS reverts that signal on
+ # its own, without touching the v1 payload.
+ #
+ # Did compression COST anything? `saved_pct` cannot answer that,
+ # and on its own it is a metric you can maximise by deleting the
+ # context. `reread_compressed` is the counter-pressure: tokens the
+ # agent had to fetch again because we compressed them away. Read
+ # the pair — a rising `saved_pct` with a rising
+ # `reread_compressed` is not a win.
+ #
+ # `reread_compressed_tokens` is a SUBSET of `reread_tokens`, not a
+ # sibling of it: the same tokens are counted in both, exactly as
+ # WasteSignals.total() documents upstream. Never add them. The
+ # ratio between them is the useful quantity — it splits re-reads
+ # the agent would have done anyway from the ones Headroom caused.
+ #
+ # `waste_turns` counts turns where ANY of these fired, re-reads
+ # included, so it is the denominator for this whole block rather
+ # than for `waste` alone.
+ "quality": {
+ "reread_tokens": self.reread_tokens,
+ "reread_compressed_tokens": self.reread_compressed_tokens,
+ # Turns where any waste signal fired at all, so the sums below
+ # have a denominator that is not "every turn".
+ "waste_turns": self.waste_turns,
+ # Waste DETECTED in the incoming request, by kind — the
+ # headroom that existed, whether or not anything took it.
+ # Sorted, like every map here, so heartbeats stay
+ # byte-comparable.
+ # A list of records, not an object keyed by waste kind. See
+ # the note on `clients` — the key set here is data-dependent
+ # too (only the kinds that fired appear), so an object would
+ # infer a different column type from one corpus to the next.
+ "waste": [
+ {"kind": kind, "tokens": tokens} for kind, tokens in sorted(self.waste.items())
+ ],
+ },
+ # Distributions for the quantities the payload otherwise reports
+ # only as totals. `counts` is len(edges) + 1; `edges` ships with
+ # every row so a reader never has to know the client version.
+ #
+ # `turn_tokens` and `overhead_ms` are observed on every turn and
+ # sum to `session.turns`. The other three are observed only where
+ # the quantity is defined (original > 0, a measured ttfb, a known
+ # message count) and sum to less.
+ "hist": {
+ name: {"edges": list(_HIST_EDGES[name]), "counts": list(counts)}
+ for name, counts in sorted(self.hists.items())
+ },
+ # The left-hand side compression has never had. `by_content` is
+ # the joint table of (what the router saw) x (what it chose) x
+ # (what that returned) — a per-shape yield, and the table a
+ # routing policy is fitted on. `by_tool` is the same question for
+ # JSON tool output, keyed by structure rather than by type.
+ #
+ # Lists of records, not objects keyed by shape, for the reason
+ # spelled out on `by_strategy`: DuckDB infers an object with few
+ # consistent keys as a STRUCT and one with many as a MAP, so an
+ # object here would change column type as the fleet meets new
+ # content. A list of records has fixed field names forever.
+ "shapes": {
+ "by_content": [
+ {
+ "content": content,
+ "strategy": strategy,
+ "n": counts[0],
+ "tokens_in": counts[1],
+ "tokens_out": counts[2],
+ }
+ for (content, strategy), counts in sorted(self.shapes.items())
+ ],
+ "by_tool": [
+ {
+ "shape": shape,
+ "n": counts[0],
+ "tokens_in": counts[1],
+ "tokens_out": counts[2],
+ }
+ for shape, counts in sorted(self.tool_shapes.items())
+ ],
+ # Set when `_fit_to_wire` had to shed rows to stay under the
+ # receiver's body cap. ALWAYS present, never conditional: an
+ # optional key would give `shapes` two different STRUCT layouts
+ # across the corpus, which is the same failure the lists above
+ # exist to avoid.
+ "truncated": False,
+ },
+ # Prompt-cache physics. `hits`/`misses` are indexed by how long the
+ # turn waited since the previous one, which makes (gap -> hit rate)
+ # an empirical survival curve for a TTL no provider documents
+ # honestly. The first turn of a session has no previous turn and is
+ # excluded from both, so hits + misses == turns - 1.
+ "cache": {
+ "gap_edges_s": list(_CACHE_GAP_EDGES),
+ "hits": list(self.cache_gap_hits),
+ "misses": list(self.cache_gap_misses),
+ "write_5m": self.cache_write_5m,
+ "write_1h": self.cache_write_1h,
+ # Turns whose write column the provider did not report and
+ # Headroom inferred (OpenAI). Without this the two are
+ # indistinguishable and the estimate reads as measurement.
+ "inferred_turns": self.cache_inferred_turns,
+ },
+ # The session's SHAPE over time rather than its totals. Buckets are
+ # log-spaced by turn index (turn 1, turns 2-3, 4-7, ... 512+), so
+ # the arrays describe how context grew and where compression
+ # started or stopped paying — the input to predicting a session's
+ # terminal size from its opening turns.
+ "trajectory": {
+ "buckets": _TRAJECTORY_BUCKETS,
+ "turns": list(self.traj_turns),
+ "tokens": list(self.traj_tokens),
+ "saved": list(self.traj_saved),
+ # Run-length encoded turn kinds from the closed `_TURN_KINDS`
+ # alphabet, e.g. "s12p3z40s2". Structural: a kind is derived
+ # from a status code and two token counts, never from content.
+ "kinds": "".join(f"{kind}{run}" for kind, run in self.kind_runs),
+ "kinds_truncated": self.kinds_truncated,
+ "msgs_max": self.msgs_max,
+ },
+ # Which harness drove the session. `headroom.stack` answers this by
+ # detection and, as its own note concedes, resolves to the literal
+ # "proxy" for most of the fleet; this is the value the handler
+ # actually identified. Turns with no identified client are absent
+ # rather than bucketed, so the values sum to <= `session.turns`.
+ # A LIST of records, not an object keyed by client. This is the
+ # same decision `by_strategy` documents, and it is not stylistic:
+ # DuckDB infers a JSON object as a STRUCT while its keys are few
+ # and identifier-shaped, and as a MAP once they are not, so the
+ # COLUMN TYPE would be a function of which harnesses happened to
+ # appear in the data. Measured on a real corpus: two harnesses
+ # infer STRUCT(claude_code, codex) and a query for
+ # `clients.cursor` fails to bind; twenty-two infer
+ # MAP(VARCHAR, BIGINT) and the same query succeeds. That is
+ # exactly the break that silently took out the transforms report,
+ # and the client vocabulary grows every time a harness is added.
+ #
+ # A list of records has fixed field names forever, so the type is
+ # the same on day one and after the thirtieth harness ships.
+ "clients": [
+ {"client": client, "n": turns} for client, turns in sorted(self.clients.items())
+ ],
+ # 4xx, kept strictly apart from `failures`. 429 is the one that
+ # matters: fleet-wide rate-limit rate per provider and hour is a
+ # live view of which upstream is degraded, which is a routing
+ # input and is invisible in a 5xx-only count.
+ "errors": {
+ "count": self.client_errors,
+ "by_status": dict(sorted(self.client_error_statuses.items())),
+ },
+ # output_shaper A/B strata, as validated enums rather than as the
+ # free-text label suffix `_fold` strips. Same instinct as that
+ # strip — the FORMAT was the problem — but the arm and the request
+ # features are what make the experiment readable at all.
+ #
+ # Model family is deliberately NOT here. It is the one component of
+ # the stratum key that names something outside the request, the
+ # existing `models` field already reports it wherever a public
+ # registry vouches for the id, and this payload has no business
+ # being a second, unvetted route for it.
+ # One list rather than four objects, for the reason on `clients`:
+ # each of these was an object whose keys are whichever stratum
+ # values the session happened to hit, so all four had a
+ # data-dependent column type. Flattened to (dimension, value,
+ # count) records they are one stable column that pivots in a
+ # GROUP BY, and a new stratum dimension adds rows instead of
+ # changing a type.
+ "strata": [
+ {"dim": dim, "value": value, "n": turns}
+ for dim, counter in (
+ ("arm", self.strata_arm),
+ ("turn_kind", self.strata_turn_kind),
+ ("input_bucket", self.strata_input_bucket),
+ ("tools", self.strata_tools),
+ )
+ for value, turns in sorted(counter.items())
+ ],
+ # Allowlisted configuration. Lets the corpus separate "compression
+ # underperformed" from "compression was switched off", which the
+ # token counts alone cannot do. Names and values are both bounded —
+ # see `_config_snapshot`.
+ # Records, not an object: every operator sets a different subset
+ # of `_CONFIG_ENV_KEYS`, so the key set is the most data-dependent
+ # of the four. Same failure, same fix.
+ "config": [
+ {"key": key, "value": value} for key, value in sorted(_config_snapshot().items())
+ ],
+ }
+
class SessionAggregator:
"""Folds per-request outcomes into one event per activity burst.
@@ -592,6 +1268,10 @@ def _fold(sess: _Session, outcome: Any, now: float, source: str = "proxy") -> No
def get(name: str, default: Any = 0) -> Any:
return getattr(outcome, name, default)
+ # Captured before `last_seen` moves: the gap between this turn and the one
+ # before it is what the cache-survival curve is indexed on. None on the
+ # first turn of a session, which has no predecessor to measure against.
+ prev_seen = sess.last_seen if sess.turns else None
sess.last_seen = now
sess.turns += 1
sess.sources[source] = sess.sources.get(source, 0) + 1
@@ -675,6 +1355,165 @@ def _fold(sess: _Session, outcome: Any, now: float, source: str = "proxy") -> No
key = str(name).split(":", 1)[0][:64]
sess.transforms[key] = sess.transforms.get(key, 0) + 1
+ # Every v1 counter is folded above and is not read below. Guarded for the
+ # same reason `payload` guards its v2 half: unguarded, a raise here aborts
+ # `_fold` before the caller's heartbeat check and costs a whole emission,
+ # which is a v1 consequence for a v2 fault.
+ try:
+ _fold_extra(sess, get, now, prev_seen, tags, status)
+ except Exception:
+ logger.debug("telemetry: v2 fold failed", exc_info=True)
+
+
+def _fold_extra(
+ sess: _Session,
+ get: Callable[..., Any],
+ now: float,
+ prev_seen: float | None,
+ tags: Any,
+ status: int,
+) -> None:
+ """Fold the schema-v2 signals. Additive by construction.
+
+ Split out of `_fold` rather than appended to it, for one reason: every v1
+ counter is finished before this runs and nothing here reads or writes one.
+ A mistake in a new signal can therefore only cost that signal. The corpus
+ has years of rows whose meaning depends on `tokens.saved` and
+ `failure_statuses` staying exactly what they have always been, and a
+ boundary is a better guarantee of that than care is.
+
+ Duck-typed through the same `get` as `_fold`, so `_McpCompression` — which
+ knows four fields — still folds without inventing the rest.
+ """
+ original = int(get("original_tokens") or 0)
+ saved = int(get("tokens_saved") or 0)
+ attempted = int(get("attempted_input_tokens") or 0)
+ is_dict_tags = isinstance(tags, dict)
+ passthrough = is_dict_tags and "passthrough_reason" in tags
+ cached = bool(get("from_response_cache", False))
+
+ # -- distributions ------------------------------------------------------
+ sess.observe("turn_tokens", original)
+ sess.observe("overhead_ms", float(get("overhead_ms", 0.0) or 0.0))
+ if original > 0:
+ # Undefined without a denominator, and piling those into the 0% bucket
+ # would report a compression failure where there was no request.
+ sess.observe("saved_pct", _pct(saved, original))
+ ttfb = float(get("ttfb_ms", 0.0) or 0.0)
+ if ttfb > 0:
+ # 0 is the convention for "not measured" on this field (non-streaming
+ # paths), not for "instant".
+ sess.observe("ttfb_ms", ttfb)
+ msgs = int(get("num_messages") or 0)
+ if msgs > 0:
+ sess.observe("msgs", msgs)
+ if msgs > sess.msgs_max:
+ sess.msgs_max = msgs
+
+ # -- regret / waste -----------------------------------------------------
+ waste = get("waste_signals", None)
+ if isinstance(waste, dict):
+ touched = False
+ for name in (*_WASTE_KEYS, "reread", "reread_compressed"):
+ try:
+ amount = int(waste.get(name, 0) or 0)
+ except (TypeError, ValueError):
+ continue
+ if amount <= 0:
+ continue
+ touched = True
+ if name == "reread":
+ sess.reread_tokens += amount
+ elif name == "reread_compressed":
+ sess.reread_compressed_tokens += amount
+ else:
+ sess.waste[name] = sess.waste.get(name, 0) + amount
+ if touched:
+ sess.waste_turns += 1
+
+ # -- cache physics ------------------------------------------------------
+ if prev_seen is not None:
+ index = _bucket(_CACHE_GAP_EDGES, now - prev_seen)
+ if int(get("cache_read_tokens") or 0) > 0:
+ sess.cache_gap_hits[index] += 1
+ else:
+ sess.cache_gap_misses[index] += 1
+ sess.cache_write_5m += int(get("cache_write_5m_tokens") or 0)
+ sess.cache_write_1h += int(get("cache_write_1h_tokens") or 0)
+ if get("cache_inferred", False):
+ sess.cache_inferred_turns += 1
+
+ # -- trajectory ---------------------------------------------------------
+ # `sess.turns` was incremented by _fold, so turn 1 lands in bucket 0.
+ index = min(sess.turns.bit_length() - 1, _TRAJECTORY_BUCKETS - 1)
+ sess.traj_turns[index] += 1
+ sess.traj_tokens[index] += original
+ sess.traj_saved[index] += saved
+ sess.append_kind(
+ _turn_kind(
+ cached=cached,
+ status=status,
+ passthrough=passthrough,
+ attempted=attempted,
+ saved=saved,
+ )
+ )
+
+ # -- harness and 4xx ----------------------------------------------------
+ client = get("client", None)
+ if client:
+ _bump(sess.clients, _client_slug(client))
+ if 400 <= status < 500:
+ sess.client_errors += 1
+ # `status` is already an int in this range, so the key is a bounded
+ # numeric string and needs no slug guard.
+ _bump(sess.client_error_statuses, str(status))
+
+ # -- A/B strata ---------------------------------------------------------
+ # Re-read rather than shared with the transforms loop above: that loop owns
+ # a v1 field and is left exactly as it was.
+ for name in get("transforms_applied", ()) or ():
+ text = str(name)
+ if text.startswith(_STRATUM_TREATMENT):
+ arm, key = "treatment", text[len(_STRATUM_TREATMENT) :]
+ elif text.startswith(_STRATUM_CONTROL):
+ arm, key = "control", text[len(_STRATUM_CONTROL) :]
+ else:
+ continue
+ _bump(sess.strata_arm, arm)
+ # family|turn_kind|input_bucket|tools. Index 0 is the model family and
+ # is deliberately dropped -- see the note in payload().
+ parts = key.split("|")
+ for position, counter in (
+ (1, sess.strata_turn_kind),
+ (2, sess.strata_input_bucket),
+ (3, sess.strata_tools),
+ ):
+ if len(parts) > position:
+ _bump(counter, _safe_slug(parts[position]))
+
+ # -- shape tables -------------------------------------------------------
+ # Drained here for the same reason `_staged_strategies` is drained in
+ # `_fold`: a compression event must not be able to open a session.
+ for shape_key, staged in _drain_staged_shapes().items():
+ counts = sess.shapes.get(shape_key)
+ if counts is None:
+ if len(sess.shapes) >= MAX_SHAPE_ROWS:
+ continue
+ counts = [0, 0, 0]
+ sess.shapes[shape_key] = counts
+ for i in range(3):
+ counts[i] += staged[i]
+ for tool_key, staged in _drain_staged_tool_shapes().items():
+ counts = sess.tool_shapes.get(tool_key)
+ if counts is None:
+ if len(sess.tool_shapes) >= MAX_SHAPE_ROWS:
+ continue
+ counts = [0, 0, 0]
+ sess.tool_shapes[tool_key] = counts
+ for i in range(3):
+ counts[i] += staged[i]
+
# --------------------------------------------------------------------------
# OTLP/HTTP JSON transport
@@ -732,30 +1571,184 @@ def build_otlp_logs(payload: dict[str, Any], resource: dict[str, str]) -> dict[s
}
+def _send(endpoint: str, body: bytes, agent: str, timeout: float, *, compress: bool) -> None:
+ """One POST. Raises `_CompressionRejected` if gzip is why it was refused."""
+ headers = {
+ "content-type": "application/json",
+ # Required, not cosmetic. urllib defaults to "Python-urllib/3.x",
+ # which Cloudflare blocks outright by browser signature (error
+ # 1010) before the request ever reaches the Worker. Combined
+ # with fire-and-forget error handling that failure mode is
+ # invisible: every upload 403s and the beacon looks healthy.
+ "user-agent": agent,
+ }
+ if compress:
+ body = gzip.compress(body, _GZIP_LEVEL)
+ # Spec-correct for OTLP/HTTP, which is what makes this work against a
+ # generic collector via HEADROOM_TELEMETRY_ENDPOINT. Our own Worker
+ # ignores it and sniffs the gzip magic number instead, because an
+ # intermediary may have decompressed the body before it arrives.
+ headers["content-encoding"] = "gzip"
+ request = urllib.request.Request(endpoint, data=body, method="POST", headers=headers)
+ try:
+ with urllib.request.urlopen(request, timeout=timeout):
+ pass
+ except urllib.error.HTTPError as err:
+ try:
+ err.close()
+ except Exception:
+ pass
+ # "Could not read this body": a Worker older than the one that learned
+ # to inflate (its JSON.parse throws on the gzip bytes and it answers
+ # 400), or a collector that never will (415). Deliberately not the
+ # whole 4xx range -- see _GZIP_REFUSED_STATUSES. 5xx is the endpoint
+ # being unwell and says nothing about gzip, so it must not disable
+ # compression forever either.
+ if compress and err.code in _GZIP_REFUSED_STATUSES:
+ raise _CompressionRejected from err
+ raise
+
+
+# What one POST may put on the wire.
+#
+# The receiver answers an oversized body with 413, and 413 is deliberately
+# absent from `_GZIP_REFUSED_STATUSES` because the uncompressed retry is larger
+# still -- so a 413 is total loss of the event, v1 counters included. The client
+# cannot discover the far end's cap (the response arrives after the body), so it
+# has to bound itself instead of hoping the two numbers match.
+#
+# 60KB sits under the OLDEST receiver still in service, whose cap is 64KB. That
+# is the number that matters, not the 256KB the current Worker allows: an
+# install upgrades on its own schedule and may be pointed at any deployment.
+#
+# Measured: schema v2 is ~8KB before the shape tables and crosses 64KB at ~116
+# rows in each, against caps of MAX_SHAPE_ROWS. So this binds only the busiest
+# sessions, and only while bodies are uncompressed -- gzip puts a saturated
+# payload at ~4KB, where the budget is unreachable and nothing is ever shed.
+_MAX_WIRE_BYTES = 60 * 1024
+
+
+def _fit_to_wire(payload: dict[str, Any], resource: dict[str, str], *, compress: bool) -> bytes:
+ """Encode `payload`, shedding shape rows until it fits `_MAX_WIRE_BYTES`.
+
+ Budgets what ARRIVES, so the check runs against the compressed length when
+ compression is on. That is what makes this self-cancelling: once gzip ships,
+ a saturated payload is ~4KB and this never trims anything.
+
+ Mutates `payload` in place when it trims. Safe because the aggregator
+ builds a fresh snapshot per emit and hands it to exactly one sink.
+
+ `shapes` is what gets shed because it is ~90% of a large body and the only
+ part that is genuinely aggregate -- the same (content x strategy) cells are
+ re-measured by every other session, so dropping the thinnest rows from one
+ of them costs resolution, not a fact. Everything else is a session-scoped
+ total that exists nowhere else. Rows are ranked by `n` so what survives is
+ what carries the most evidence.
+
+ Never raises: a failure here degrades to sending the untrimmed body, which
+ is exactly what would have been sent without this function.
+ """
+
+ def encode(candidate: dict[str, Any]) -> tuple[bytes, int]:
+ raw = json.dumps(build_otlp_logs(candidate, resource), separators=(",", ":")).encode()
+ return raw, len(gzip.compress(raw, _GZIP_LEVEL)) if compress else len(raw)
+
+ raw, wire = encode(payload)
+ try:
+ if wire <= _MAX_WIRE_BYTES:
+ return raw
+ shapes = payload.get("shapes")
+ if not isinstance(shapes, dict):
+ return raw
+ by_content = sorted(shapes.get("by_content") or [], key=lambda r: -r.get("n", 0))
+ by_tool = sorted(shapes.get("by_tool") or [], key=lambda r: -r.get("n", 0))
+ if not (by_content or by_tool):
+ return raw
+
+ # Largest surviving prefix that fits. Binary search rather than dropping
+ # a row at a time: a saturated payload is 320 rows, and each probe
+ # re-encodes ~80KB.
+ fitted = False
+ lo, hi = 0, max(len(by_content), len(by_tool))
+ while lo < hi:
+ mid = (lo + hi + 1) // 2
+ shapes["by_content"] = by_content[:mid]
+ shapes["by_tool"] = by_tool[:mid]
+ shapes["truncated"] = True
+ _, wire = encode(payload)
+ if wire <= _MAX_WIRE_BYTES:
+ fitted, lo = True, mid
+ else:
+ hi = mid - 1
+ if fitted:
+ # Re-sorted back into the canonical order the payload emits. The
+ # ranking above is only a selection rule; letting it leak into the
+ # wire format would mean the row order of `by_content` silently
+ # depended on whether the body happened to need trimming.
+ shapes["by_content"] = sorted(
+ by_content[:lo],
+ key=lambda r: (r.get("content", ""), r.get("strategy", "")),
+ )
+ shapes["by_tool"] = sorted(by_tool[:lo], key=lambda r: r.get("shape", ""))
+ shapes["truncated"] = True
+ # Re-encoded rather than reusing the probe's bytes: reordering the
+ # same records cannot change the length, so this still fits.
+ return encode(payload)[0]
+ # Even zero shape rows did not fit. Send the stripped body anyway: the
+ # v1 counters are the point, and a 413 would lose them too.
+ shapes["by_content"] = []
+ shapes["by_tool"] = []
+ shapes["truncated"] = True
+ return encode(payload)[0]
+ except Exception:
+ logger.debug("telemetry: wire-size trim failed", exc_info=True)
+ return raw
+
+
def _post_blocking(payload: dict[str, Any], timeout: float = _POST_TIMEOUT_S) -> None:
endpoint = os.environ.get("HEADROOM_TELEMETRY_ENDPOINT", DEFAULT_ENDPOINT)
+ # Hoisted above the encode: the wire budget measures what actually leaves,
+ # so it has to know whether that will be compressed or plain.
+ compress = _gzip_enabled()
try:
from headroom._version import get_version
- data = json.dumps(
- build_otlp_logs(payload, resource_attributes()), separators=(",", ":")
- ).encode()
- request = urllib.request.Request(
- endpoint,
- data=data,
- method="POST",
- headers={
- "content-type": "application/json",
- # Required, not cosmetic. urllib defaults to "Python-urllib/3.x",
- # which Cloudflare blocks outright by browser signature (error
- # 1010) before the request ever reaches the Worker. Combined
- # with fire-and-forget error handling that failure mode is
- # invisible: every upload 403s and the beacon looks healthy.
- "user-agent": f"headroom-beacon/{get_version()}",
- },
- )
- with urllib.request.urlopen(request, timeout=timeout):
- pass
+ resource = resource_attributes()
+ body = _fit_to_wire(payload, resource, compress=compress)
+ agent = f"headroom-beacon/{get_version()}"
+ except Exception:
+ logger.debug("telemetry: session POST failed", exc_info=True)
+ return
+
+ # Compress by default, and fall back the moment the endpoint says it cannot
+ # read it. This is what makes the rollout order forgiving: shipping this
+ # client against a Worker that predates gzip costs one extra round trip per
+ # process and no data, instead of silently dropping every upload.
+ #
+ # That failure mode is not hypothetical here. Uploads are fire-and-forget
+ # and the status is otherwise ignored, so "every POST is refused" and
+ # "everything is fine" look identical from inside the process -- exactly
+ # how the Cloudflare 1010 user-agent block went unnoticed.
+ if compress:
+ try:
+ _send(endpoint, body, agent, timeout, compress=True)
+ return
+ except _CompressionRejected:
+ logger.debug("telemetry: endpoint refused gzip; falling back", exc_info=True)
+ _disable_gzip()
+ # Re-budget before resending. `body` was measured compressed, and
+ # the same bytes plain can be ~6x larger -- the one case where the
+ # fallback could turn a refusal into a 413 and lose the event it
+ # was added to save.
+ try:
+ body = _fit_to_wire(payload, resource, compress=False)
+ except Exception:
+ logger.debug("telemetry: re-trim after gzip refusal failed", exc_info=True)
+ except Exception:
+ logger.debug("telemetry: session POST failed", exc_info=True)
+ return
+ try:
+ _send(endpoint, body, agent, timeout, compress=False)
except Exception:
logger.debug("telemetry: session POST failed", exc_info=True)
@@ -926,6 +1919,178 @@ def record_compression(strategy: str, original_tokens: int, compressed_tokens: i
counts[2] += after
+def _stage(store: dict[Any, list[int]], key: Any, before: int, after: int) -> None:
+ """Add one (n, tokens_in, tokens_out) observation to a staged shape table.
+
+ Shared by the two `record_*_shape` entry points, which differ only in what
+ they key on. Assumes the caller holds `_staged_lock`.
+ """
+ counts = store.get(key)
+ if counts is None:
+ if len(store) >= MAX_SHAPE_ROWS:
+ return
+ counts = [0, 0, 0]
+ store[key] = counts
+ counts[0] += 1
+ counts[1] += before
+ counts[2] += after
+
+
+def _clean_pair(original_tokens: Any, compressed_tokens: Any) -> tuple[int, int] | None:
+ """Validate one before/after token pair, or None if it is not usable.
+
+ Clamped exactly as `record_compression` clamps: a compressor that emits
+ more than it received is a bug, and letting `out` exceed `in` would surface
+ downstream as negative savings rather than as the bug it is.
+ """
+ try:
+ before = int(original_tokens or 0)
+ after = int(compressed_tokens or 0)
+ except (TypeError, ValueError):
+ return None
+ if before <= 0:
+ return None
+ return before, min(max(after, 0), before)
+
+
+def record_content_shapes(observations: Iterable[tuple[str, str, int, int]]) -> None:
+ """Beacon entry point for a batch of routing decisions from one compress().
+
+ Batched, not one call per decision, because `_staged_lock` is also held by
+ `record_compression` on the same executor thread and the note there is
+ explicit that the contention is real: the router observes once per content
+ section per request. Taking the lock once per compress() rather than once
+ per section keeps this addition off that critical path.
+
+ Each observation is ``(content_type, strategy, original_tokens,
+ compressed_tokens)``. Both strings are enum values in code -- `ContentType`
+ has nine members and `CompressionStrategy` a handful -- but both arrive as
+ free strings, so both are slugged before they can reach the wire.
+ """
+ from headroom.telemetry.beacon import is_beacon_enabled
+
+ if not is_beacon_enabled():
+ return
+ cleaned: list[tuple[tuple[str, str], int, int]] = []
+ for content_type, strategy, original_tokens, compressed_tokens in observations:
+ pair = _clean_pair(original_tokens, compressed_tokens)
+ if pair is None:
+ continue
+ cleaned.append(((_safe_slug(content_type), _safe_slug(strategy)), *pair))
+ if not cleaned:
+ return
+ with _staged_lock:
+ for key, before, after in cleaned:
+ _stage(_staged_shapes, key, before, after)
+
+
+def record_content_shape(
+ content_type: str, strategy: str, original_tokens: int, compressed_tokens: int
+) -> None:
+ """Beacon entry point for one routing decision, keyed by what was routed.
+
+ `record_compression` already reports what each strategy yielded. This adds
+ the term that was missing: what it was handed. Without a content axis the
+ corpus can rank compressors against each other but cannot say which one to
+ pick for a given input, which is the decision the router actually makes.
+
+ Both arguments are enum values in code — `ContentType` has nine members and
+ `CompressionStrategy` a handful — but both arrive here as free strings, so
+ both are slugged before they can reach the wire.
+
+ Single-observation form of :func:`record_content_shapes`, which is what the
+ router actually calls. Same discipline as everything else in this module:
+ off by default, cheap when off, never raises, and never touches the
+ aggregator lock.
+ """
+ record_content_shapes(((content_type, strategy, original_tokens, compressed_tokens),))
+
+
+def record_tool_shape(signature: Any, original_tokens: int, compressed_tokens: int) -> None:
+ """Beacon entry point for one tool-output compression, keyed by structure.
+
+ TOIN's docstring has always promised a network effect -- "more users ->
+ more compression events -> better aggregated fields" -- and there has never
+ been a transport, so every install learns alone and `import_patterns` has
+ no source to import from. This is that transport, minus the part that
+ cannot safely leave a machine.
+
+ What ships is the STRUCTURAL DESCRIPTOR only: a bucketed field count, a
+ nesting depth, and the boolean shape flags `ToolSignature` documents as
+ "pattern indicators (without revealing actual field names)". What does not
+ ship is `structure_hash`, which is a SHA256 over sorted field names and
+ types -- a function of the tool's data, and the same objection that
+ (rightly) kept `conversation_key_from_body` off the wire applies to it
+ unchanged. Nor do `field_retrieval_frequency` or `common_query_patterns`,
+ the two `ToolPattern` fields that carry real content.
+
+ A descriptor also needs no k-anonymity machinery to be safe, because it is
+ not invertible in the first place: thousands of distinct tools collapse
+ into a few dozen shape classes, which is exactly the clustering the
+ recommendations want anyway.
+
+ `signature` is read through `getattr` and never required to be a real
+ `ToolSignature`, so this cannot break a caller that passes something else.
+ """
+ from headroom.telemetry.beacon import is_beacon_enabled
+
+ if not is_beacon_enabled():
+ return
+ pair = _clean_pair(original_tokens, compressed_tokens)
+ if pair is None:
+ return
+ key = _tool_shape_key(signature)
+ if key is None:
+ return
+ with _staged_lock:
+ _stage(_staged_tool_shapes, key, *pair)
+
+
+# Shape flags, in a fixed order so one structure always produces one key.
+# Letters, not the field names, and distinct from each other: `status` takes
+# `u` because `s` is already `score`.
+_SHAPE_FLAGS = (
+ ("has_arrays", "a"),
+ ("has_nested_objects", "n"),
+ ("has_id_like_field", "i"),
+ ("has_score_like_field", "s"),
+ ("has_timestamp_like_field", "t"),
+ ("has_status_like_field", "u"),
+ ("has_error_like_field", "e"),
+ ("has_message_like_field", "m"),
+)
+
+
+def _tool_shape_key(signature: Any) -> str | None:
+ """Structural descriptor for one tool output, as a bounded slug.
+
+ Shaped like ``f16d3_anit``: field count rounded DOWN to a power of two,
+ nesting depth, then the shape flags that are set. Rounding the field count
+ is what makes the key a class rather than a fingerprint -- 17 fields and 31
+ fields are the same shape for every purpose this serves.
+
+ Returns None when the object is not a signature, rather than a key built
+ from defaults, so a bad caller contributes nothing instead of contributing
+ noise.
+ """
+ try:
+ fields = int(getattr(signature, "field_count", 0) or 0)
+ depth = int(getattr(signature, "max_depth", 0) or 0)
+ except (TypeError, ValueError):
+ return None
+ if fields <= 0:
+ return None
+ bucket = min(1 << (fields.bit_length() - 1), 64)
+ flags = "".join(
+ letter for attribute, letter in _SHAPE_FLAGS if getattr(signature, attribute, False)
+ )
+ key = f"f{bucket}d{min(max(depth, 0), 9)}" + (f"_{flags}" if flags else "")
+ # Belt and braces: the key is built from integers and literals above, so it
+ # already matches, but nothing reaches the wire without passing the same
+ # validator every other slug does.
+ return _safe_slug(key)
+
+
def record_stack(slug: str) -> None:
"""Beacon entry point for one request's stack slug.
@@ -990,6 +2155,26 @@ def _drain_staged_strategies() -> dict[str, list[int]]:
return drained
+def _drain_staged_shapes() -> dict[tuple[str, str], list[int]]:
+ """Take every staged (content_type, strategy) observation."""
+ with _staged_lock:
+ if not _staged_shapes:
+ return {}
+ drained = {key: counts[:] for key, counts in _staged_shapes.items()}
+ _staged_shapes.clear()
+ return drained
+
+
+def _drain_staged_tool_shapes() -> dict[str, list[int]]:
+ """Take every staged tool-structure observation."""
+ with _staged_lock:
+ if not _staged_tool_shapes:
+ return {}
+ drained = {key: counts[:] for key, counts in _staged_tool_shapes.items()}
+ _staged_tool_shapes.clear()
+ return drained
+
+
def record_outcome(outcome: Any) -> None:
"""Beacon entry point, called from the proxy's outcome funnel.
@@ -1426,6 +2611,112 @@ def demo() -> None:
# A malformed outcome must not raise either.
SessionAggregator(emitted.append).record(object(), now=4000.0)
+ # --- schema v2 ---------------------------------------------------------
+ # The full v2 surface is covered by tests/test_beacon_schema_v2.py. What
+ # belongs here is the part that has to hold for the module on its own: the
+ # additions are additive, and the two things they read that nobody had read
+ # before — the environment, and the output_shaper stratum label — cannot
+ # carry anything out with them.
+ _staged_shapes.clear()
+ _staged_tool_shapes.clear()
+
+ class V2(FakeOutcome):
+ original_tokens = 1000
+ attempted_input_tokens = 400
+ tokens_saved = 300
+ ttfb_ms = 300.0
+ num_messages = 30
+ # Hyphenated, as `classify_client` really returns it.
+ client = "claude-code"
+ cache_write_5m_tokens = 80
+ cache_write_1h_tokens = 20
+ waste_signals = {"reread": 900, "reread_compressed": 400, "json_bloat": 50}
+ # Carries a model family, a turn kind and a size bucket. Only the last
+ # two may leave, and neither the label nor the family may.
+ transforms_applied = ("output_shaper:stratum:sonnet|tool_result|l|tools", "crush")
+
+ class V2Limited(V2):
+ status_code = 429
+
+ v2_out: list[dict[str, Any]] = []
+ v2 = SessionAggregator(v2_out.append)
+ v2.record(V2(), now=10_000.0)
+ v2.record(V2Limited(), now=10_045.0)
+ v2.record(V2(), now=10_090.0)
+ v2.flush_all()
+ row = v2_out[-1]
+
+ # v1 is untouched by all of it. A 4xx in particular must not reach the
+ # failure counters, which have been 5xx-only in every row of the corpus.
+ assert row["failures"] == 0 and row["failure_statuses"] == {}, row
+ assert row["errors"] == {"count": 1, "by_status": {"429": 1}}, row["errors"]
+ assert row["compression"]["transforms"] == {"output_shaper": 3, "crush": 3}
+
+ # The regret label: saved_pct alone cannot say whether compression cost the
+ # agent anything, and this is the counter-pressure on it.
+ assert row["quality"]["reread_compressed_tokens"] == 1200, row["quality"]
+ assert row["quality"]["waste"] == [{"kind": "json_bloat", "tokens": 150}], (
+ "waste kinds are allowlisted"
+ )
+
+ # Distributions ship their bucket edges, so a row is readable without
+ # knowing which client version wrote it.
+ assert row["hist"]["turn_tokens"]["edges"] == list(_HIST_EDGES["turn_tokens"])
+ assert sum(row["hist"]["turn_tokens"]["counts"]) == 3
+ assert len(row["hist"]["saved_pct"]["counts"]) == len(_HIST_EDGES["saved_pct"]) + 1
+
+ # Cumulative like every v1 counter, so max(seq) still reconstructs the
+ # session and a dropped heartbeat still costs nothing.
+ assert sum(row["trajectory"]["turns"]) == row["session"]["turns"]
+ assert row["trajectory"]["kinds"] == "s1e1s1", row["trajectory"]
+ assert set(row["trajectory"]["kinds"]) - set("0123456789") <= _TURN_KINDS
+
+ # The first turn has no predecessor, so the cache curve counts turns - 1.
+ assert sum(row["cache"]["hits"]) + sum(row["cache"]["misses"]) == 2
+
+ # Strata reach the wire as validated enums; the model family does not.
+ # Hyphenated client ids must survive; _safe_slug alone would drop them.
+ assert row["clients"] == [{"client": "claude_code", "n": 3}], row["clients"]
+ # Lists of records, never objects keyed by a value that came from data —
+ # an object's COLUMN TYPE in DuckDB is a function of which keys showed up.
+ assert {"dim": "arm", "value": "treatment", "n": 3} in row["strata"], row["strata"]
+ assert {"dim": "input_bucket", "value": "l", "n": 3} in row["strata"], row["strata"]
+ assert isinstance(row["config"], list) and isinstance(row["quality"]["waste"], list)
+ assert "sonnet" not in json.dumps(row), "the stratum label leaked a model family"
+
+ # Structural descriptor only, never TOIN's structure_hash.
+ assert _tool_shape_key(object()) is None
+ assert (
+ _tool_shape_key(
+ type(
+ "Sig",
+ (),
+ {"field_count": 17, "max_depth": 3, "has_arrays": True, "has_id_like_field": True},
+ )()
+ )
+ == "f16d3_ai"
+ )
+
+ # The environment is read through an allowlist, and every value that gets
+ # past it is slugged — so a variable holding a path or a URL reports
+ # "other" rather than its contents.
+ _cfg_prev = {k: os.environ.get(k) for k in ("HEADROOM_MODE", "HEADROOM_KOMPRESS_ENDPOINT")}
+ try:
+ os.environ["HEADROOM_MODE"] = "/Users/someone/private"
+ os.environ["HEADROOM_KOMPRESS_ENDPOINT"] = "https://secret.internal/v1"
+ snapshot = _config_snapshot()
+ assert snapshot.get("mode") == "other", snapshot
+ assert "kompress_endpoint" not in snapshot, snapshot
+ finally:
+ for _k, _v in _cfg_prev.items():
+ if _v is None:
+ os.environ.pop(_k, None)
+ else:
+ os.environ[_k] = _v
+
+ _staged_shapes.clear()
+ _staged_tool_shapes.clear()
+
print("ok")
diff --git a/headroom/telemetry/toin.py b/headroom/telemetry/toin.py
index 34fb3579d..f3675ab94 100644
--- a/headroom/telemetry/toin.py
+++ b/headroom/telemetry/toin.py
@@ -565,6 +565,23 @@ class ToolIntelligenceNetwork:
model_family: Target model family (`claude-3-5`, `gpt-4o`, …).
Defaults to `DEFAULT_MODEL_FAMILY` when not provided.
"""
+ # The beacon's copy of this event is taken BEFORE the enabled check.
+ # TOIN's own store is gated on HEADROOM_TELEMETRY, which is opt-in and
+ # therefore off across almost the whole fleet -- so recording after the
+ # gate would mean the network in "Tool Output Intelligence Network"
+ # only ever sees the installs that least need it.
+ #
+ # Costs nothing extra: `tool_signature` was already built by the
+ # caller, and `record_tool_shape` reads a handful of its integer
+ # attributes and no hash at all. Off by default and never raises, like
+ # every other beacon entry point.
+ try:
+ from headroom.telemetry.session import record_tool_shape
+
+ record_tool_shape(tool_signature, original_tokens, compressed_tokens)
+ except Exception: # pragma: no cover - telemetry must never break a request
+ logger.debug("beacon: tool shape recording failed", exc_info=True)
+
# HIGH FIX: Check enabled FIRST to avoid computing structure_hash if disabled
# This saves CPU when TOIN is turned off
if not self._config.enabled:
diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.py
index 629f9eaf7..ccac516b1 100644
--- a/headroom/transforms/content_router.py
+++ b/headroom/transforms/content_router.py
@@ -1454,6 +1454,47 @@ class RoutingDecision:
return self.compressed_tokens / self.original_tokens
+def _record_beacon_shapes(routing_log: list[RoutingDecision]) -> None:
+ """Report (content_type -> strategy -> yield) for one compress() call.
+
+ `RoutingDecision` already carries the content type the detector assigned
+ alongside the strategy that was picked for it, so the joint table costs
+ nothing to measure -- it has simply never been reported anywhere. The
+ beacon's `by_strategy` sees the strategy and its yield but not the input,
+ which leaves it able to rank compressors and unable to say which one suits
+ a given piece of content.
+
+ Deliberately NOT routed through `CompressionObserver`: that protocol is
+ implemented outside this repo as well, and widening it would break every
+ such implementation. This is a direct call to a function that is off by
+ default, cheap when off, and cannot raise.
+
+ Runs whether or not an observer is installed, since the beacon path and the
+ Prometheus path are independent -- an install with no observer still
+ reports token totals, and would otherwise report an empty shape table
+ against them.
+ """
+ if not routing_log:
+ return
+ try:
+ from ..telemetry.session import record_content_shapes
+
+ # One call, not one per decision: the beacon's staging lock is shared
+ # with `record_compression` on this same executor thread, and the note
+ # there is explicit that the contention is real.
+ record_content_shapes(
+ (
+ decision.content_type.value,
+ decision.strategy.value,
+ decision.original_tokens,
+ decision.compressed_tokens,
+ )
+ for decision in routing_log
+ )
+ except Exception as e: # pragma: no cover - defensive
+ logger.debug("beacon shape recording failed (non-fatal): %s", e)
+
+
@dataclass
class RouterCompressionResult:
"""Result from ContentRouter with routing metadata.
@@ -2561,6 +2602,7 @@ class ContentRouter(Transform):
anyway, swallow at debug level. Compression already succeeded;
a buggy observer must not turn a 200 into a 500.
"""
+ _record_beacon_shapes(result.routing_log)
if self._observer is None:
return
for d in result.routing_log:
diff --git a/tests/test_beacon_gzip.py b/tests/test_beacon_gzip.py
new file mode 100644
index 000000000..9d2c5ca2b
--- /dev/null
+++ b/tests/test_beacon_gzip.py
@@ -0,0 +1,424 @@
+"""Beacon upload compression, and the fallback that makes it safe to ship.
+
+A schema-v2 event is ~8KB of repetitive JSON and gzips ~6x, which is worth more
+than every schema trim available put together. The risk is not the compression;
+it is that uploads are fire-and-forget, so an endpoint that cannot read a
+gzipped body looks exactly like an endpoint that is working. These tests cover
+the compression, and then the three ways it is allowed to stop.
+
+The server here re-implements worker.js's sniff-and-inflate in Python. It is not
+the Worker, and cannot prove `DecompressionStream` behaves — but it does pin the
+wire contract the two halves have to agree on, which is the part that would
+otherwise only be checked in production.
+"""
+
+from __future__ import annotations
+
+import gzip
+import itertools
+import json
+import threading
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from typing import Any
+
+import pytest
+
+from headroom.telemetry import session as S
+
+
+class _Recorder(BaseHTTPRequestHandler):
+ """Stands in for worker.js's fetch(): sniff magic bytes, inflate, parse."""
+
+ def do_POST(self) -> None: # noqa: N802 - BaseHTTPRequestHandler API
+ raw = self.rfile.read(int(self.headers.get("content-length", 0)))
+ box = self.server.box # type: ignore[attr-defined]
+
+ if box.get("too_large"):
+ self.send_response(413)
+ self.end_headers()
+ box["over"] += 1
+ return
+ if box.get("refuse_gzip") and raw[:2] == b"\x1f\x8b":
+ # A Worker that predates the inflate path: JSON.parse throws on the
+ # gzip bytes and it answers 400.
+ self.send_response(400)
+ self.end_headers()
+ box["refused"] += 1
+ return
+ if box.get("fail_5xx"):
+ self.send_response(503)
+ self.end_headers()
+ return
+
+ body = gzip.decompress(raw) if raw[:2] == b"\x1f\x8b" else raw
+ box["events"].append(json.loads(body))
+ box["encodings"].append(self.headers.get("content-encoding"))
+ box["wire_bytes"].append(len(raw))
+ self.send_response(204)
+ self.end_headers()
+
+ def log_message(self, *args: Any) -> None:
+ pass
+
+
+def collector(monkeypatch):
+ server = HTTPServer(("127.0.0.1", 0), _Recorder)
+ server.box = { # type: ignore[attr-defined]
+ "events": [],
+ "encodings": [],
+ "wire_bytes": [],
+ "refused": 0,
+ "over": 0,
+ }
+ thread = threading.Thread(target=server.serve_forever, daemon=True)
+ thread.start()
+ host, port = server.server_address
+ monkeypatch.setenv("HEADROOM_TELEMETRY_ENDPOINT", f"http://{host}:{port}/v1/logs")
+ monkeypatch.setenv("HEADROOM_BEACON", "on")
+ monkeypatch.delenv("DO_NOT_TRACK", raising=False)
+ # gzip ships opt-in (_GZIP_DEFAULT is False for the staged rollout), so the
+ # tests that exercise the transport have to turn it on explicitly. The
+ # default itself is locked by test_gzip_is_opt_in_by_default below, which
+ # deliberately does NOT use this fixture's environment.
+ monkeypatch.setenv("HEADROOM_BEACON_GZIP", "1")
+ # Compression disables itself process-wide on refusal; reset between tests.
+ monkeypatch.setattr(S, "_gzip_supported", True)
+ yield server.box # type: ignore[attr-defined]
+ server.shutdown()
+ server.server_close()
+
+
+class Outcome:
+ provider = "anthropic"
+ model = "gpt-4o"
+ original_tokens = 120_000
+ attempted_input_tokens = 40_000
+ optimized_tokens = 90_000
+ output_tokens = 1_200
+ tokens_saved = 30_000
+ cache_read_tokens = 60_000
+ status_code = 200
+ total_latency_ms = 4_200.0
+ overhead_ms = 48.0
+ ttfb_ms = 900.0
+ num_messages = 140
+ client = "claude-code"
+ transforms_applied: tuple[str, ...] = ("crush",)
+ tags: dict[str, Any] = {}
+ waste_signals = {"reread": 900, "reread_compressed": 400}
+
+
+def sample_payload() -> dict[str, Any]:
+ rows: list[dict[str, Any]] = []
+ agg = S.SessionAggregator(rows.append)
+ for index in range(40):
+ agg.record(Outcome(), now=1000.0 + index * 45)
+ agg.flush_all()
+ return rows[-1]
+
+
+def test_payload_survives_the_round_trip_byte_for_byte(collector):
+ """Compression must be invisible above the transport: what the collector
+ parses has to be exactly what the aggregator built."""
+ payload = sample_payload()
+ S._post_blocking(payload)
+ assert len(collector["events"]) == 1
+ received = collector["events"][0]
+ expected = S.build_otlp_logs(payload, S.resource_attributes())
+ body = received["resourceLogs"][0]["scopeLogs"][0]["logRecords"][0]["body"]
+ want = expected["resourceLogs"][0]["scopeLogs"][0]["logRecords"][0]["body"]
+ assert body == want, "the payload changed in transit"
+
+
+def test_it_is_actually_compressed_and_declared(collector):
+ S._post_blocking(sample_payload())
+ assert collector["encodings"] == ["gzip"], "Content-Encoding was not declared"
+ plain = len(
+ json.dumps(
+ S.build_otlp_logs(sample_payload(), S.resource_attributes()), separators=(",", ":")
+ ).encode()
+ )
+ sent = collector["wire_bytes"][0]
+ assert sent < plain / 3, f"only compressed {plain / sent:.1f}x ({sent} vs {plain} B)"
+
+
+def test_a_worker_that_cannot_inflate_costs_one_retry_not_the_data(collector):
+ """The rollout-order safety net. Shipping this client against a Worker that
+ predates the inflate path must not lose events — it must notice, fall back,
+ and stop compressing."""
+ collector["refuse_gzip"] = True
+ S._post_blocking(sample_payload())
+ assert collector["refused"] == 1, "the compressed attempt never happened"
+ assert len(collector["events"]) == 1, "the event was lost instead of retried"
+ assert collector["encodings"] == [None], "the retry was not uncompressed"
+
+ # ...and it must not keep paying that retry for the rest of the process.
+ S._post_blocking(sample_payload())
+ assert collector["refused"] == 1, "it tried gzip again after being refused"
+ assert len(collector["events"]) == 2
+ assert collector["encodings"] == [None, None]
+
+
+def test_a_5xx_does_not_disable_compression(collector):
+ """A sick endpoint says nothing about whether it understands gzip. Treating
+ a 503 as "no gzip here" would permanently give up the saving on the first
+ upstream blip."""
+ collector["fail_5xx"] = True
+ S._post_blocking(sample_payload())
+ assert S._gzip_enabled() is True
+
+ collector["fail_5xx"] = False
+ S._post_blocking(sample_payload())
+ assert collector["encodings"] == ["gzip"]
+
+
+def test_the_kill_switch_works(collector, monkeypatch):
+ monkeypatch.setenv("HEADROOM_BEACON_GZIP", "off")
+ S._post_blocking(sample_payload())
+ assert collector["encodings"] == [None]
+ assert len(collector["events"]) == 1
+
+
+def test_a_dead_endpoint_still_never_raises(monkeypatch):
+ monkeypatch.setenv("HEADROOM_TELEMETRY_ENDPOINT", "http://127.0.0.1:1/v1/logs")
+ monkeypatch.setenv("HEADROOM_BEACON", "on")
+ S._post_blocking(sample_payload(), timeout=0.25)
+
+
+def test_the_wire_contract_worker_js_must_satisfy():
+ """What worker.js sniffs for. If this changes, the Worker's `bytes[0] ===
+ 0x1f && bytes[1] === 0x8b` check stops matching and every upload 400s."""
+ body = json.dumps(
+ S.build_otlp_logs(sample_payload(), S.resource_attributes()), separators=(",", ":")
+ ).encode()
+ compressed = gzip.compress(body, S._GZIP_LEVEL)
+ assert compressed[0] == 0x1F and compressed[1] == 0x8B, "not gzip magic"
+ assert gzip.decompress(compressed) == body, "not standard gzip"
+ # And the Worker's arriving-body cap must still be comfortable.
+ assert len(compressed) < 64 * 1024
+
+
+def test_a_413_must_not_trigger_the_uncompressed_retry(collector):
+ """413 means the body was too big. Answering that by re-sending the SAME
+ payload uncompressed — six times larger — is guaranteed to 413 again, and
+ would permanently switch off the compression that was the only thing
+ keeping the request under the cap. Only "I cannot read this encoding"
+ (400/415) may fall back."""
+ collector["too_large"] = True
+ S._post_blocking(sample_payload())
+ assert collector["over"] == 1, "it should not have retried at all"
+ assert S._gzip_enabled() is True, "a 413 disabled compression"
+
+
+def test_415_does_fall_back(collector, monkeypatch):
+ """The other honest 'I cannot read this body' answer, from a collector that
+ rejects the encoding outright rather than failing to parse it."""
+
+ class Refuse415(_Recorder):
+ def do_POST(self): # noqa: N802
+ raw = self.rfile.read(int(self.headers.get("content-length", 0)))
+ box = self.server.box
+ if raw[:2] == b"\x1f\x8b":
+ self.send_response(415)
+ self.end_headers()
+ box["refused"] += 1
+ return
+ box["events"].append(json.loads(raw))
+ box["encodings"].append(self.headers.get("content-encoding"))
+ self.send_response(204)
+ self.end_headers()
+
+ collector_server = HTTPServer(("127.0.0.1", 0), Refuse415)
+ collector_server.box = {"events": [], "encodings": [], "refused": 0}
+ threading.Thread(target=collector_server.serve_forever, daemon=True).start()
+ host, port = collector_server.server_address
+ monkeypatch.setenv("HEADROOM_TELEMETRY_ENDPOINT", f"http://{host}:{port}/v1/logs")
+ try:
+ S._post_blocking(sample_payload())
+ assert collector_server.box["refused"] == 1
+ assert collector_server.box["encodings"] == [None], "no uncompressed retry"
+ assert S._gzip_enabled() is False
+ finally:
+ collector_server.shutdown()
+ collector_server.server_close()
+
+
+def test_the_gzip_gate_never_raises(monkeypatch):
+ """`_gzip_enabled` is called before _post_blocking's try, from a bare
+ daemon-thread target and from an atexit handler — both places where an
+ escaping exception reaches the user's terminal."""
+ import headroom.telemetry.beacon as beacon_module
+
+ monkeypatch.delattr(beacon_module, "_OFF_VALUES", raising=False)
+ assert S._gzip_enabled() is False, "it must degrade, not raise"
+ # ...and the upload path still completes.
+ monkeypatch.setenv("HEADROOM_TELEMETRY_ENDPOINT", "http://127.0.0.1:1/v1/logs")
+ S._post_blocking(sample_payload(), timeout=0.25)
+
+
+def test_gzip_is_opt_in_by_default(monkeypatch):
+ """The staged rollout: schema v2 ships without the new transport.
+
+ Locks the default so enabling gzip is a deliberate edit to `_GZIP_DEFAULT`
+ (or an operator setting the variable), never a side effect of touching this
+ module. While it holds, the Worker's `inflate()` is unreachable -- the
+ receiver sniffs magic bytes, and nothing produces them.
+ """
+ monkeypatch.delenv("HEADROOM_BEACON_GZIP", raising=False)
+ monkeypatch.setattr(S, "_gzip_supported", True)
+ assert S._GZIP_DEFAULT is False, "gzip must stay staged until v2 is proven"
+ assert S._gzip_enabled() is False, "an unset variable must not compress"
+
+
+def test_an_unrecognised_gzip_value_does_not_enable_it(monkeypatch):
+ """A typo degrades to the proven transport, not to the new one."""
+ monkeypatch.setattr(S, "_gzip_supported", True)
+ for value in ("yess", "maybe", "2", " "):
+ monkeypatch.setenv("HEADROOM_BEACON_GZIP", value)
+ assert S._gzip_enabled() is False, f"{value!r} enabled compression"
+
+
+def test_the_operator_can_opt_in(monkeypatch):
+ """...and every documented on-value works, so the opt-in is discoverable."""
+ monkeypatch.setattr(S, "_gzip_supported", True)
+ for value in ("1", "on", "true", "yes", "enable", "enabled", "ON", " 1 "):
+ monkeypatch.setenv("HEADROOM_BEACON_GZIP", value)
+ assert S._gzip_enabled() is True, f"{value!r} did not enable compression"
+
+
+# --------------------------------------------------------------- wire budget --
+#
+# The receiver answers an oversized body with 413, and 413 is deliberately not a
+# reason to retry, so an event that exceeds the cap is lost outright -- v1
+# counters included. These cover the client bounding itself instead.
+
+_CONTENT = [f"ct{i:02d}" for i in range(10)]
+_STRATEGIES = [f"st{i:02d}" for i in range(13)]
+
+
+def _saturated(n_shape: int = 160, n_tool: int = 160, turns: int = 200) -> S._Session:
+ """A session whose shape tables are at their caps."""
+ sess = S._Session(sid="a" * 32, started=0.0, last_seen=0.0)
+ sess.turns = turns
+ sess.tokens_saved = 987_654
+ combos = list(itertools.product(_CONTENT, _STRATEGIES))[:n_shape]
+ for i, (content, strategy) in enumerate(combos):
+ sess.shapes[(content, strategy)] = [i + 1, 222_222, 33_333]
+ for i in range(n_tool):
+ sess.tool_shapes[f"tool_shape_{i:03d}|d{i % 6}|w{i % 8}"] = [i + 1, 222_222, 33_333]
+ sess.models.add("claude_sonnet_5")
+ sess.providers.add("anthropic")
+ sess.clients["claude_code"] = turns
+ return sess
+
+
+def _resource() -> dict[str, str]:
+ res = S.resource_attributes(create_install_id=False)
+ res.setdefault("headroom.install_id", "0" * 32)
+ return res
+
+
+def _body_of(raw: bytes) -> dict[str, Any]:
+ """Unwrap one OTLP body back to plain JSON -- the inverse of _any_value."""
+
+ def unwrap(value: dict[str, Any]) -> Any:
+ if "kvlistValue" in value:
+ return {kv["key"]: unwrap(kv["value"]) for kv in value["kvlistValue"]["values"]}
+ if "arrayValue" in value:
+ return [unwrap(v) for v in value["arrayValue"].get("values", [])]
+ if "intValue" in value:
+ return int(value["intValue"])
+ for key in ("stringValue", "boolValue", "doubleValue"):
+ if key in value:
+ return value[key]
+ return None
+
+ record = json.loads(raw)["resourceLogs"][0]["scopeLogs"][0]["logRecords"][0]
+ return unwrap(record["body"])
+
+
+def test_an_oversized_plain_body_is_trimmed_to_fit():
+ """Uncompressed schema v2 crosses 64KB at ~116 rows per table; caps are higher.
+
+ Without this the busiest sessions -- the most informative ones -- would be
+ exactly the ones the receiver refuses.
+ """
+ payload = _saturated().payload("heartbeat")
+ raw = S._fit_to_wire(payload, _resource(), compress=False)
+ assert len(raw) <= 64 * 1024, f"{len(raw)} B exceeds the 64KB receiver cap"
+ assert _body_of(raw)["shapes"]["truncated"] is True
+
+
+def test_trimming_never_costs_a_v1_counter():
+ """The whole point of shedding shapes is that nothing else is shed."""
+ payload = _saturated().payload("heartbeat")
+ expected = payload["tokens"]["saved"]
+ body = _body_of(S._fit_to_wire(payload, _resource(), compress=False))
+ assert body["shapes"]["truncated"] is True, "nothing was shed; test proves nothing"
+ assert body["tokens"]["saved"] == expected
+ assert body["session"]["turns"] == 200
+
+
+def test_the_trim_keeps_the_rows_carrying_the_most_evidence():
+ """Rows are ranked by `n`, so a trimmed table loses resolution, not signal."""
+ payload = _saturated().payload("heartbeat")
+ kept = _body_of(S._fit_to_wire(payload, _resource(), compress=False))["shapes"]
+ counts = [row["n"] for row in kept["by_content"]]
+ assert counts, "everything was dropped"
+ # 130 (content x strategy) rows carry n = 1..130. Keeping the top means the
+ # thinnest survivor still beats what was dropped.
+ assert min(counts) > 1, "the trim kept the least-used rows"
+
+
+def test_a_trimmed_table_keeps_the_canonical_row_order():
+ """Ranking is a selection rule, not a wire format."""
+ payload = _saturated().payload("heartbeat")
+ kept = _body_of(S._fit_to_wire(payload, _resource(), compress=False))["shapes"]
+ assert kept["truncated"] is True, "nothing was shed; test proves nothing"
+ by_content = [(r["content"], r["strategy"]) for r in kept["by_content"]]
+ assert by_content == sorted(by_content), "row order depended on trimming"
+
+
+def test_truncated_is_always_present_even_when_nothing_is_shed():
+ """An optional key would give `shapes` two STRUCT layouts in the corpus."""
+ payload = _saturated(n_shape=2, n_tool=2).payload("heartbeat")
+ raw = S._fit_to_wire(payload, _resource(), compress=False)
+ shapes = _body_of(raw)["shapes"]
+ assert shapes["truncated"] is False
+ assert len(shapes["by_content"]) == 2, "a body under budget was trimmed anyway"
+
+
+def test_compression_makes_the_budget_unreachable():
+ """Self-cancelling: once gzip ships, a saturated payload is ~4KB."""
+ payload = _saturated().payload("heartbeat")
+ raw = S._fit_to_wire(payload, _resource(), compress=True)
+ shapes = _body_of(raw)["shapes"]
+ assert shapes["truncated"] is False, "gzip should leave the tables intact"
+ assert len(gzip.compress(raw, S._GZIP_LEVEL)) <= S._MAX_WIRE_BYTES
+
+
+def test_the_gzip_fallback_rebudgets_before_resending(collector, monkeypatch):
+ """The refusal path resends plain -- and plain is ~6x larger.
+
+ Without re-measuring, the fallback added to SAVE an event from a Worker that
+ cannot inflate would hand that Worker a body over its cap instead, turning a
+ 400 into a 413 and losing the event after all.
+ """
+ monkeypatch.setenv("HEADROOM_BEACON_GZIP", "1")
+ monkeypatch.setattr(S, "_gzip_supported", True)
+ collector["refuse_gzip"] = True
+ S._post_blocking(_saturated().payload("heartbeat"), timeout=5.0)
+ assert collector["refused"] == 1, "gzip was not attempted first"
+ assert collector["events"], "the fallback never arrived"
+ # 64KB is the cap on the OLDEST receiver still in service -- a literal on
+ # purpose. Asserting against _MAX_WIRE_BYTES would restate the code under
+ # test, and would keep passing if that budget were ever raised past what a
+ # deployed Worker accepts.
+ assert collector["wire_bytes"][-1] <= 64 * 1024, (
+ f"fell back with {collector['wire_bytes'][-1]} B, over the 64KB receiver cap"
+ )
+ # The collector keeps the OTLP envelope; unwrap it back to the payload.
+ sent = _body_of(json.dumps(collector["events"][-1]).encode())
+ assert sent["tokens"]["saved"], "the fallback lost the v1 counters"
+ assert sent["shapes"]["truncated"] is True, "the plain resend was not re-budgeted"
diff --git a/tests/test_beacon_schema_v2.py b/tests/test_beacon_schema_v2.py
new file mode 100644
index 000000000..950b605c9
--- /dev/null
+++ b/tests/test_beacon_schema_v2.py
@@ -0,0 +1,906 @@
+"""Beacon schema v2: the new signals, and proof the old ones did not move.
+
+The corpus has years of rows whose meaning depends on the v1 payload staying
+exactly what it has always been. Schema v2 is additive, and this file is where
+that claim is mechanical rather than asserted: `test_v1_*` locks the shape and
+the arithmetic of every pre-existing field, `test_v2_*` covers what was added,
+and `test_allowlist_covers_every_emitted_key` enforces the deploy-ordering rule
+that a key the worker does not allow is a key that is silently discarded.
+"""
+
+from __future__ import annotations
+
+import gzip
+import json
+import re
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+from headroom.telemetry import session as S
+
+# ---------------------------------------------------------------------------
+# The v1 payload, spelled out. Changing anything in this literal is a schema
+# break for every consumer of the existing corpus, and should be very hard to
+# do by accident.
+# ---------------------------------------------------------------------------
+
+V1_TOP = {
+ "schema_version",
+ "session",
+ "tokens",
+ "rates",
+ "compression",
+ "skips",
+ "sources",
+ "providers",
+ "models",
+ "failures",
+ "failure_statuses",
+}
+
+V1_NESTED = {
+ "session": {"id", "seq", "duration_s", "turns", "ended", "final"},
+ "tokens": {
+ "original",
+ "attempted",
+ "input",
+ "output",
+ "saved",
+ "tool_saved",
+ "cache_read",
+ "cache_write",
+ "uncached",
+ },
+ "rates": {
+ "saved_pct",
+ "eligible_pct",
+ "yield_pct",
+ "all_layers_saved_pct",
+ "all_layers_yield_pct",
+ "cache_read_pct",
+ "overhead_pct",
+ },
+ "compression": {
+ "transforms",
+ "by_strategy",
+ "overhead_ms_total",
+ "latency_ms_total",
+ "overhead_ms_per_turn",
+ "latency_ms_per_turn",
+ "passthrough_turns",
+ "response_cache_hits",
+ },
+}
+
+V2_TOP = {
+ "quality",
+ "hist",
+ "shapes",
+ "cache",
+ "trajectory",
+ "clients",
+ "errors",
+ "strata",
+ "config",
+}
+
+
+class Outcome:
+ """A fully-populated RequestOutcome stand-in. `_fold` is duck-typed."""
+
+ provider = "anthropic"
+ model = "gpt-4o"
+ original_tokens = 1000
+ attempted_input_tokens = 400
+ optimized_tokens = 700
+ provider_input_tokens = 700
+ output_tokens = 20
+ tokens_saved = 300
+ cache_read_tokens = 500
+ cache_write_tokens = 100
+ cache_write_5m_tokens = 80
+ cache_write_1h_tokens = 20
+ uncached_input_tokens = 400
+ cache_inferred = False
+ from_response_cache = False
+ status_code = 200
+ total_latency_ms = 1000.0
+ overhead_ms = 50.0
+ ttfb_ms = 300.0
+ num_messages = 30
+ client = "claude-code" # real ids are hyphenated; see test_client_ids_survive
+ transforms_applied: tuple[str, ...] = ("crush",)
+ tags: dict[str, Any] = {}
+ waste_signals: dict[str, int] | None = None
+
+
+def emit(
+ *outcomes: Any, start: float = 1000.0, step: float = 45.0, source: str = "proxy"
+) -> dict[str, Any]:
+ """Fold a sequence of outcomes into one session and return its final row."""
+ rows: list[dict[str, Any]] = []
+ agg = S.SessionAggregator(rows.append)
+ for index, outcome in enumerate(outcomes):
+ agg.record(outcome, now=start + index * step, source=source)
+ agg.flush_all()
+ return rows[-1]
+
+
[email protected](autouse=True)
+def _clean_staging():
+ """Staging is module state; a leak across tests would double-count."""
+ S._staged_strategies.clear()
+ S._staged_stacks.clear()
+ S._staged_shapes.clear()
+ S._staged_tool_shapes.clear()
+ yield
+ S._staged_strategies.clear()
+ S._staged_stacks.clear()
+ S._staged_shapes.clear()
+ S._staged_tool_shapes.clear()
+
+
+def beacon_on(monkeypatch):
+ """conftest turns the beacon off for hermeticity; the record_* entry points
+ short-circuit on that, so anything testing them has to turn it back on."""
+ monkeypatch.setenv("HEADROOM_BEACON", "on")
+ monkeypatch.delenv("DO_NOT_TRACK", raising=False)
+ monkeypatch.delenv("HEADROOM_OFFLINE", raising=False)
+
+
+# ---------------------------------------------------------------------------
+# v1: nothing moved
+# ---------------------------------------------------------------------------
+
+
+def test_v1_keys_all_present_and_unchanged():
+ payload = emit(Outcome())
+ assert V1_TOP <= set(payload), f"v1 key(s) dropped: {V1_TOP - set(payload)}"
+ for section, expected in V1_NESTED.items():
+ assert expected <= set(payload[section]), (
+ f"{section} lost key(s): {expected - set(payload[section])}"
+ )
+
+
+def test_v1_arithmetic_is_byte_for_byte_what_it_was():
+ """The five ratios the product is judged on, plus the counters they derive
+ from. These values predate schema v2 and must survive it exactly."""
+ payload = emit(Outcome())
+ assert payload["tokens"] == {
+ "original": 1000,
+ "attempted": 400,
+ "input": 700,
+ "output": 20,
+ "saved": 300,
+ "tool_saved": 0,
+ "cache_read": 500,
+ "cache_write": 100,
+ "uncached": 400,
+ }
+ assert payload["rates"]["saved_pct"] == 30.0
+ assert payload["rates"]["eligible_pct"] == 40.0
+ assert payload["rates"]["yield_pct"] == 75.0
+ assert payload["rates"]["cache_read_pct"] == 50.0
+ assert payload["rates"]["overhead_pct"] == 5.0
+ assert payload["compression"]["transforms"] == {"crush": 1}
+ assert payload["sources"] == {"proxy": 1}
+ assert payload["providers"] == ["anthropic"]
+
+
+def test_v1_failures_stay_5xx_only_when_4xx_is_present():
+ """The single most dangerous change in v2: 4xx is now counted, and it must
+ be counted somewhere `failures` cannot see. Widening `failures` would have
+ silently redefined it — a session that rate-limited would start reading as
+ a session that failed.
+ """
+
+ class RateLimited(Outcome):
+ status_code = 429
+
+ class Overloaded(Outcome):
+ status_code = 529
+
+ payload = emit(Outcome(), RateLimited(), RateLimited(), Overloaded())
+ assert payload["failures"] == 1, "a 4xx leaked into the 5xx failure count"
+ assert payload["failure_statuses"] == {"529": 1}
+ assert payload["errors"] == {"count": 2, "by_status": {"429": 2}}
+
+
+def test_v1_transforms_still_collapse_the_stratum_suffix():
+ """v2 reads the stratum label too, but the v1 `transforms` map must still
+ see only the prefix — the suffix carries a model family."""
+
+ class Shaped(Outcome):
+ transforms_applied = ("output_shaper:stratum:sonnet|tool_result|l|tools", "crush")
+
+ payload = emit(Shaped())
+ assert payload["compression"]["transforms"] == {"output_shaper": 1, "crush": 1}
+
+
+def test_v1_cumulative_dedupe_still_holds_with_v2_fields():
+ """The corpus dedupes on max(seq) per (install, session) and never sums
+ across heartbeats. Every v2 field has to be cumulative for that to keep
+ working."""
+ rows: list[dict[str, Any]] = []
+ agg = S.SessionAggregator(rows.append, idle_s=900.0, flush_s=100.0)
+ for index in range(6):
+ agg.record(Outcome(), now=7000.0 + index * 50)
+ agg.flush_all()
+
+ assert len({row["session"]["id"] for row in rows}) == 1
+ assert [row["session"]["seq"] for row in rows] == list(range(len(rows)))
+ for field in ("turns",):
+ values = [row["session"][field] for row in rows]
+ assert values == sorted(values), f"session.{field} is not monotonic"
+ # Every v2 counter must climb or hold, never reset.
+ for row_a, row_b in zip(rows, rows[1:]):
+ assert row_b["quality"]["reread_tokens"] >= row_a["quality"]["reread_tokens"]
+ assert sum(row_b["trajectory"]["turns"]) >= sum(row_a["trajectory"]["turns"])
+ for name, hist in row_b["hist"].items():
+ assert sum(hist["counts"]) >= sum(row_a["hist"][name]["counts"]), name
+ # The last row alone reconstructs the session.
+ assert rows[-1]["session"]["turns"] == 6
+ assert sum(rows[-1]["trajectory"]["turns"]) == 6
+
+
+def test_mcp_shim_still_folds_without_the_new_fields():
+ """`_McpCompression` knows five fields. v2 reads a dozen more through the
+ same duck-typed `get`, and must not require any of them."""
+ payload = emit(
+ S._McpCompression(
+ original_tokens=800,
+ attempted_input_tokens=800,
+ optimized_tokens=200,
+ tokens_saved=600,
+ ),
+ source="mcp",
+ )
+ assert payload["sources"] == {"mcp": 1}
+ assert payload["rates"]["yield_pct"] == 75.0
+ assert payload["trajectory"]["kinds"] == "s1"
+ assert payload["clients"] == []
+
+
+def test_a_malformed_outcome_still_cannot_raise():
+ rows: list[dict[str, Any]] = []
+ agg = S.SessionAggregator(rows.append)
+ agg.record(object(), now=1.0)
+ agg.flush_all()
+
+
+# ---------------------------------------------------------------------------
+# v2: the new signals
+# ---------------------------------------------------------------------------
+
+
+def test_quality_reports_the_regret_label():
+ class Regretful(Outcome):
+ waste_signals = {
+ "reread": 900,
+ "reread_compressed": 400,
+ "json_bloat": 50,
+ "base64": 7,
+ }
+
+ payload = emit(Regretful(), Regretful(), Outcome())
+ assert payload["quality"]["reread_tokens"] == 1800
+ assert payload["quality"]["reread_compressed_tokens"] == 800
+ assert payload["quality"]["waste"] == [
+ {"kind": "base64", "tokens": 14},
+ {"kind": "json_bloat", "tokens": 100},
+ ]
+ assert payload["quality"]["waste_turns"] == 2, "the clean turn was counted"
+
+
+def test_waste_keys_are_allowlisted_not_copied():
+ class Sneaky(Outcome):
+ waste_signals = {"json_bloat": 10, "some_future_key": 99, "/etc/passwd": 5}
+
+ payload = emit(Sneaky())
+ assert payload["quality"]["waste"] == [{"kind": "json_bloat", "tokens": 10}]
+
+
+def test_histograms_ship_their_own_edges_and_count_defined_samples():
+ class Small(Outcome):
+ original_tokens = 500
+ tokens_saved = 0
+ ttfb_ms = 0.0 # 0 means unmeasured, not instant
+
+ payload = emit(Outcome(), Outcome(), Small())
+ turn_tokens = payload["hist"]["turn_tokens"]
+ assert turn_tokens["edges"] == [1000, 4000, 16000, 64000, 128000, 256000]
+ assert len(turn_tokens["counts"]) == len(turn_tokens["edges"]) + 1
+ assert sum(turn_tokens["counts"]) == 3, "observed on every turn"
+ assert turn_tokens["counts"][0] == 1, "the 500-token turn is under the first edge"
+ assert sum(payload["hist"]["ttfb_ms"]["counts"]) == 2, "unmeasured ttfb excluded"
+ # 30% saved -> the [20, 40) bucket, which is index 4.
+ assert payload["hist"]["saved_pct"]["counts"][4] == 2
+
+
+def test_cache_gap_curve_excludes_the_first_turn():
+ class Cold(Outcome):
+ cache_read_tokens = 0
+
+ payload = emit(Outcome(), Outcome(), Cold(), step=45.0)
+ hits = payload["cache"]["hits"]
+ misses = payload["cache"]["misses"]
+ assert sum(hits) + sum(misses) == 2, "first turn has no predecessor to measure"
+ assert payload["cache"]["gap_edges_s"] == [30.0, 60.0, 120.0, 300.0, 600.0, 1800.0]
+ # A 45s gap lands in the [30, 60) bucket.
+ assert hits[1] == 1 and misses[1] == 1
+ assert payload["cache"]["write_5m"] == 240
+ assert payload["cache"]["write_1h"] == 60
+
+
+def test_trajectory_buckets_are_log_spaced_and_kinds_are_run_length_encoded():
+ class Bypassed(Outcome):
+ attempted_input_tokens = 0
+ tokens_saved = 0
+ tags = {"passthrough_reason": "bypass_header"}
+
+ payload = emit(*([Outcome()] * 4), Bypassed(), Bypassed(), *([Outcome()] * 3))
+ assert payload["trajectory"]["turns"] == [1, 2, 4, 2, 0, 0, 0, 0, 0, 0]
+ assert payload["trajectory"]["kinds"] == "s4p2s3"
+ assert payload["trajectory"]["kinds_truncated"] is False
+ assert payload["trajectory"]["msgs_max"] == 30
+ # And the v1 passthrough counter is untouched by any of it.
+ assert payload["compression"]["passthrough_turns"] == 2
+
+
+def test_trajectory_kind_alphabet_is_closed():
+ class Cached(Outcome):
+ from_response_cache = True
+
+ class Barren(Outcome):
+ tokens_saved = 0
+
+ class Ineligible(Outcome):
+ attempted_input_tokens = 0
+ tokens_saved = 0
+
+ class Failed(Outcome):
+ status_code = 500
+
+ class Limited(Outcome):
+ status_code = 429
+
+ payload = emit(Cached(), Failed(), Limited(), Barren(), Outcome(), Ineligible())
+ kinds = payload["trajectory"]["kinds"]
+ assert kinds == "c1f1e1z1s1o1"
+ assert set(re.findall(r"[a-z]", kinds)) <= S._TURN_KINDS
+
+
+def test_trajectory_string_is_capped():
+ outcomes = []
+ for index in range(S.MAX_TRAJECTORY_RUNS * 3):
+ cls = Outcome if index % 2 else type("Alt", (Outcome,), {"status_code": 500})
+ outcomes.append(cls())
+ payload = emit(*outcomes)
+ assert payload["trajectory"]["kinds_truncated"] is True
+ assert len(re.findall(r"[a-z]", payload["trajectory"]["kinds"])) == S.MAX_TRAJECTORY_RUNS
+
+
+def test_clients_and_strata(beacon_on):
+ class Shaped(Outcome):
+ transforms_applied = ("output_shaper:stratum:sonnet|tool_result|l|tools",)
+
+ class Control(Outcome):
+ client = "claude-vscode"
+ transforms_applied = ("output_shaper:control:opus|user|s|notools",)
+
+ class Anonymous(Outcome):
+ client = None
+
+ payload = emit(Shaped(), Control(), Anonymous())
+ assert payload["clients"] == [
+ {"client": "claude_code", "n": 1},
+ {"client": "claude_vscode", "n": 1},
+ ], "unknown client not bucketed"
+ strata = {(r["dim"], r["value"]): r["n"] for r in payload["strata"]}
+ assert strata[("arm", "treatment")] == 1 and strata[("arm", "control")] == 1
+ assert strata[("turn_kind", "tool_result")] == 1 and strata[("turn_kind", "user")] == 1
+ assert strata[("input_bucket", "l")] == 1 and strata[("input_bucket", "s")] == 1
+ assert strata[("tools", "tools")] == 1 and strata[("tools", "notools")] == 1
+ # Model family is the one stratum component that must never ship.
+ assert "sonnet" not in json.dumps(payload)
+ assert "opus" not in json.dumps(payload)
+
+
+def test_client_ids_survive_slugging():
+ """Every harness `classify_client` can return has to round-trip, or the
+ field reports "other" for the clients that matter. Asserted against the
+ real vocabulary rather than a value invented here — the first version of
+ this test used "claude_code", which is not an id the proxy ever emits, and
+ so passed while `claude-code` was being discarded.
+ """
+ from headroom.proxy.auth_mode import CLIENT_UA_MAP
+
+ known = sorted({value for _, value in CLIENT_UA_MAP})
+ assert "claude-code" in known, "the vocabulary moved; this test needs updating"
+
+ outcomes = [type(f"C{i}", (Outcome,), {"client": name})() for i, name in enumerate(known)]
+ clients = {r["client"]: r["n"] for r in emit(*outcomes)["clients"]}
+ assert "other" not in clients, (
+ f"a real client id was discarded: {sorted(set(known) - set(clients))}"
+ )
+ assert len(clients) == len(known)
+ assert clients["claude_code"] == 1
+
+
+def test_client_slug_still_rejects_junk():
+ payload = emit(type("Junk", (Outcome,), {"client": "../../etc/passwd"})())
+ assert payload["clients"] == [{"client": "other", "n": 1}]
+
+
+def test_content_shape_table(beacon_on):
+ S.record_content_shape("search", "smart_crusher", 1000, 400)
+ S.record_content_shape("search", "smart_crusher", 500, 300)
+ S.record_content_shape("diff", "passthrough", 800, 800)
+ payload = emit(Outcome())
+ rows = {(r["content"], r["strategy"]): r for r in payload["shapes"]["by_content"]}
+ assert rows[("search", "smart_crusher")] == {
+ "content": "search",
+ "strategy": "smart_crusher",
+ "n": 2,
+ "tokens_in": 1500,
+ "tokens_out": 700,
+ }
+ assert rows[("diff", "passthrough")]["tokens_in"] == 800
+ assert isinstance(payload["shapes"]["by_content"], list), "a list, never a map"
+
+
+def test_content_shape_cannot_open_a_session(beacon_on):
+ rows: list[dict[str, Any]] = []
+ agg = S.SessionAggregator(rows.append)
+ S.record_content_shape("search", "smart_crusher", 1000, 400)
+ agg.flush_all()
+ assert rows == [], "a compression event invented a session"
+
+
+def test_content_shape_slugs_free_strings(beacon_on):
+ S.record_content_shape("../../etc/passwd", "SELECT * FROM x", 100, 50)
+ payload = emit(Outcome())
+ assert payload["shapes"]["by_content"][0]["content"] == "other"
+ assert payload["shapes"]["by_content"][0]["strategy"] == "other"
+
+
+def test_tool_shape_is_a_descriptor_never_a_hash(beacon_on):
+ class Signature:
+ structure_hash = "deadbeefcafe0123456789ab"
+ field_count = 17
+ max_depth = 3
+ has_arrays = True
+ has_nested_objects = True
+ has_id_like_field = True
+ has_score_like_field = False
+ has_timestamp_like_field = True
+ has_status_like_field = False
+ has_error_like_field = False
+ has_message_like_field = False
+
+ S.record_tool_shape(Signature(), 5000, 1200)
+ payload = emit(Outcome())
+ assert payload["shapes"]["by_tool"] == [
+ {"shape": "f16d3_anit", "n": 1, "tokens_in": 5000, "tokens_out": 1200}
+ ]
+ assert "deadbeef" not in json.dumps(payload), "structure_hash reached the wire"
+
+
+def test_tool_shape_rejects_a_non_signature(beacon_on):
+ S.record_tool_shape(object(), 5000, 1200)
+ S.record_tool_shape(None, 5000, 1200)
+ payload = emit(Outcome())
+ assert payload["shapes"]["by_tool"] == []
+
+
+def test_shape_tables_are_capped(beacon_on):
+ """The cap is a guard against a caller inventing keys, not a budget — see
+ test_content_shape_cap_exceeds_its_vocabulary for the other side."""
+ for index in range(S.MAX_SHAPE_ROWS + 10):
+ S.record_content_shape(f"c{index}", "smart_crusher", 100, 50)
+ payload = emit(Outcome())
+ assert len(payload["shapes"]["by_content"]) == S.MAX_SHAPE_ROWS
+
+
+def test_config_is_an_allowlist_and_every_value_is_slugged(monkeypatch):
+ monkeypatch.setenv("HEADROOM_MODE", "optimize")
+ monkeypatch.setenv("HEADROOM_KOMPRESS_ENDPOINT", "https://secret.acme.internal/v1")
+ monkeypatch.setenv("HEADROOM_PROXY_TOKEN", "sk-live-abc123")
+ monkeypatch.setenv("HEADROOM_SAVINGS_PROFILE", "/Users/someone/private/path")
+ payload = emit(Outcome())
+ config = {r["key"]: r["value"] for r in payload["config"]}
+ assert config["mode"] == "optimize"
+ assert config["savings_profile"] == "other", "a path shipped verbatim"
+ assert "kompress_endpoint" not in config
+ assert "proxy_token" not in config
+ wire = json.dumps(payload)
+ assert "acme" not in wire and "sk-live" not in wire and "/Users/" not in wire
+
+
+def test_config_omits_unset_variables(monkeypatch):
+ for name in S._CONFIG_ENV_KEYS:
+ monkeypatch.delenv(name, raising=False)
+ monkeypatch.setenv("HEADROOM_DEDUPE", " ")
+ assert emit(Outcome())["config"] == []
+
+
+def test_a_v2_payload_failure_cannot_lose_the_v1_event():
+ """The v2 sections claim they cannot cost a v1 number. Built inline in one
+ dict literal they could: any raise aborts the literal and drops the whole
+ event, `tokens.saved` included."""
+ from unittest.mock import patch
+
+ rows: list[dict[str, Any]] = []
+ agg = S.SessionAggregator(rows.append)
+ with patch.object(S, "_config_snapshot", side_effect=RuntimeError("boom")):
+ agg.record(Outcome(), now=1000.0)
+ agg.flush_all()
+ assert rows, "a v2 failure dropped the entire session event"
+ assert rows[0]["tokens"]["saved"] == 300, "v1 counters survived intact"
+ assert "config" not in rows[0], "the failed v2 section is absent, not partial"
+
+
+def test_a_v2_fold_failure_cannot_cost_a_v1_counter_or_a_heartbeat():
+ from unittest.mock import patch
+
+ rows: list[dict[str, Any]] = []
+ agg = S.SessionAggregator(rows.append, idle_s=900.0, flush_s=100.0)
+ with patch.object(S, "_fold_extra", side_effect=RuntimeError("boom")):
+ for index in range(4):
+ agg.record(Outcome(), now=7000.0 + index * 60)
+ agg.flush_all()
+ assert len(rows) >= 2, "a v2 fold failure suppressed the heartbeat"
+ assert rows[-1]["session"]["turns"] == 4
+ assert rows[-1]["tokens"]["saved"] == 1200
+
+
+# ---------------------------------------------------------------------------
+# client and receiver must agree
+# ---------------------------------------------------------------------------
+
+
+def test_allowlist_covers_every_emitted_key():
+ """The worker drops any top-level key not in ALLOWED_KEYS, permanently and
+ silently. A client that emits a key the worker has not been taught is a
+ signal that looks healthy and stores nothing, so the two files have to be
+ checked against each other rather than by eye.
+ """
+ worker = (Path(__file__).resolve().parents[1] / "deploy/beacon/worker.js").read_text()
+ match = re.search(r"const ALLOWED_KEYS = \[(.*?)\n\];", worker, re.S)
+ assert match, "ALLOWED_KEYS not found in worker.js"
+ without_comments = re.sub(r"//[^\n]*", "", match.group(1))
+ allowed = set(re.findall(r"'([a-z_]+)'", without_comments))
+
+ emitted = set(emit(Outcome()))
+ missing = emitted - allowed
+ assert not missing, f"worker.js would silently discard: {sorted(missing)}"
+ assert V1_TOP <= allowed, "a v1 key was removed from the allowlist"
+ assert V2_TOP <= allowed
+
+
+def test_schema_version_was_bumped():
+ assert S.SCHEMA_VERSION == 2
+ assert emit(Outcome())["schema_version"] == 2
+
+
+def test_worst_case_payload_fits_the_receivers_body_limit():
+ """The client must not be able to build a body the Worker will refuse.
+
+ Not a formality. The cap is checked on what ARRIVES, so an uncompressed
+ upload -- the kill switch is set, or the gzip fallback fired -- is what has
+ to fit. And a 413 correctly does NOT trigger the uncompressed retry (see
+ the gzip tests), so a body over the cap is simply lost.
+
+ Read from worker.js rather than hardcoded, so raising one and not the other
+ fails here instead of in production.
+ """
+ worker = (Path(__file__).resolve().parents[1] / "deploy/beacon/worker.js").read_text()
+ match = re.search(r"const MAX_BODY_BYTES = (\d+) \* 1024;", worker)
+ assert match, "MAX_BODY_BYTES not found in worker.js"
+ cap = int(match.group(1)) * 1024
+
+ # Every bounded table driven to its cap at once.
+ for index in range(S.MAX_SHAPE_ROWS + 20):
+ S.record_content_shape(f"content{index}", f"strategy{index}", 9000, 3000)
+ S.record_tool_shape(
+ type("Sig", (), {"field_count": 8, "max_depth": index % 10, "has_arrays": True})(),
+ 5000,
+ 1200,
+ )
+ for index in range(S.MAX_STRATEGIES + 5):
+ S.record_compression(f"strategy{index}", 9000, 3000)
+
+ outcomes = []
+ for index in range(1200):
+ outcomes.append(
+ type(
+ f"O{index}",
+ (Outcome,),
+ {
+ "status_code": 400 + (index % 99),
+ "client": f"harness_{index % 70}",
+ "transforms_applied": (
+ f"output_shaper:stratum:m|kind{index % 40}|b{index % 9}|tools",
+ ),
+ "tags": {"passthrough_reason": f"reason{index % 20}"},
+ "waste_signals": dict.fromkeys(
+ (*S._WASTE_KEYS, "reread", "reread_compressed"), 9
+ ),
+ },
+ )()
+ )
+ payload = emit(*outcomes)
+ body = json.dumps(
+ S.build_otlp_logs(payload, S.resource_attributes()), separators=(",", ":")
+ ).encode()
+ assert len(body) < cap, (
+ f"worst-case OTLP body is {len(body):,} B against a {cap:,} B cap — the "
+ "Worker would 413 it and it would not be retried"
+ )
+ # Gzipped it is a rounding error, which is the normal path.
+ assert len(gzip.compress(body, S._GZIP_LEVEL)) < cap // 10
+
+
+# ---------------------------------------------------------------------------
+# headroom telemetry --show
+# ---------------------------------------------------------------------------
+
+
+def test_telemetry_show_prints_the_real_schema_not_a_mockup():
+ """The command exists so a user can verify what leaves their machine
+ without reading two source files and trusting they match the installed
+ copy. That only holds if it renders the same payload the beacon builds —
+ so it is asserted against the live key set, not against a fixture."""
+ from click.testing import CliRunner
+
+ from headroom.cli.main import main as cli
+
+ result = CliRunner().invoke(cli, ["telemetry", "--show", "--json"])
+ assert result.exit_code == 0, result.output
+ report = json.loads(result.output)
+ assert report["schema_version"] == S.SCHEMA_VERSION
+ assert V1_TOP <= set(report["session_event"])
+ assert V2_TOP <= set(report["session_event"])
+ assert set(report["session_event"]) == set(emit(Outcome())), (
+ "--show has drifted from the payload the beacon actually sends"
+ )
+
+
+def test_telemetry_show_reports_the_beacon_state(monkeypatch):
+ from click.testing import CliRunner
+
+ from headroom.cli.main import main as cli
+
+ monkeypatch.setenv("HEADROOM_BEACON", "off")
+ off = CliRunner().invoke(cli, ["telemetry", "--show", "--json"])
+ assert json.loads(off.output)["beacon_enabled"] is False
+
+ monkeypatch.setenv("HEADROOM_BEACON", "on")
+ monkeypatch.delenv("DO_NOT_TRACK", raising=False)
+ on = CliRunner().invoke(cli, ["telemetry", "--show", "--json"])
+ assert json.loads(on.output)["beacon_enabled"] is True
+
+
+def test_telemetry_command_does_not_mint_an_install_id(tmp_path, monkeypatch):
+ """Inspecting telemetry — very plausibly on the way to switching it off —
+ must not be the thing that gives a machine its identifier. `install_id()`
+ creates and persists one as a side effect, so this command reads the file
+ directly instead."""
+ from click.testing import CliRunner
+
+ from headroom.cli.main import main as cli
+
+ monkeypatch.setenv("HEADROOM_WORKSPACE_DIR", str(tmp_path))
+ monkeypatch.setenv("HOME", str(tmp_path))
+ # The id is memoised in a module global once anything has read it, so the
+ # cache has to be cleared or this test reads the developer's real id and
+ # never exercises the create path at all.
+ monkeypatch.setattr(S, "_install_id", None)
+
+ # BOTH paths. `--show` is the one that matters and the one that regressed:
+ # filtering the id out of the rendered report is not enough, because
+ # resource_attributes() mints it while building that report.
+ for argv in (["telemetry"], ["telemetry", "--show"], ["telemetry", "--json"]):
+ monkeypatch.setattr(S, "_install_id", None)
+ result = CliRunner().invoke(cli, argv)
+ assert result.exit_code == 0, result.output
+ assert not list(tmp_path.rglob("install_id")), f"{argv} created an install id"
+ assert "none yet" in CliRunner().invoke(cli, ["telemetry"]).output
+
+
+def test_clients_map_is_capped_against_a_hostile_header():
+ """`classify_client` returns the `X-Client` header verbatim when a caller
+ sets one, so this field is attacker-influenced. Junk already collapses to
+ "other"; the cap is what stops a caller minting a new valid-looking slug
+ per request and growing the payload without bound."""
+ outcomes = [
+ type(f"C{i}", (Outcome,), {"client": f"harness_{i}"})() for i in range(S.MAX_SHAPES + 25)
+ ]
+ clients = emit(*outcomes)["clients"]
+ assert len(clients) == S.MAX_SHAPES
+
+
+# ---------------------------------------------------------------------------
+# shape stability
+# ---------------------------------------------------------------------------
+
+
+def key_shape(node: Any, path: str = "") -> set[str]:
+ """Every key path in a payload, with list contents collapsed.
+
+ Numeric key segments are normalised, because a map keyed by a number is
+ safe: DuckDB cannot read "429" as a struct field name, so it infers MAP —
+ which is what the v1 `failure_statuses` column has always been across the
+ whole corpus. Identifier-shaped keys are the dangerous ones.
+ """
+ shape: set[str] = set()
+ if isinstance(node, dict):
+ for key, value in node.items():
+ name = "<num>" if str(key).isdigit() else key
+ shape.add(f"{path}.{name}")
+ shape |= key_shape(value, f"{path}.{name}")
+ elif isinstance(node, list):
+ for item in node:
+ shape |= key_shape(item, f"{path}[]")
+ return shape
+
+
+def test_payload_shape_does_not_depend_on_the_data():
+ """The invariant behind every "list of records, not an object" note in
+ payload(): two sessions with nothing in common must produce the SAME set of
+ key paths.
+
+ An object keyed by something that came from the data breaks this, and the
+ consequence is not cosmetic — DuckDB infers such an object as a STRUCT
+ while its keys are few and identifier-shaped and as a MAP once they are
+ not, so the COLUMN TYPE becomes a function of the contents. Measured on the
+ real corpus: two harnesses gave STRUCT(claude_code, codex) and a query for
+ `clients.cursor` failed to bind; twenty-two gave MAP(VARCHAR, BIGINT) and
+ the same query ran. That is the break that took out the transforms report.
+
+ This test is the general guard, so a future field cannot reintroduce it.
+ """
+
+ class Left(Outcome):
+ client = "claude-code"
+ status_code = 200
+ transforms_applied = ("output_shaper:stratum:sonnet|tool_result|l|tools",)
+ waste_signals = {"json_bloat": 50, "reread": 10}
+
+ class Right(Outcome):
+ client = "aider"
+ status_code = 200
+ transforms_applied = ("output_shaper:control:opus|new_user_ask|xs|notools",)
+ waste_signals = {"base64": 7, "whitespace": 3, "repetition": 11}
+
+ import os
+
+ os.environ["HEADROOM_MODE"] = "optimize"
+ try:
+ left = key_shape(emit(Left()))
+ finally:
+ os.environ.pop("HEADROOM_MODE", None)
+ os.environ["HEADROOM_DEDUPE"] = "on"
+ os.environ["HEADROOM_LOSSLESS"] = "on"
+ try:
+ right = key_shape(emit(Right()))
+ finally:
+ os.environ.pop("HEADROOM_DEDUPE", None)
+ os.environ.pop("HEADROOM_LOSSLESS", None)
+
+ assert left == right, (
+ "payload shape varies with its contents — these paths differ, which "
+ f"means a column type will too: {sorted(left ^ right)}"
+ )
+
+
+def test_the_fields_that_carry_open_vocabularies_are_lists():
+ """Belt and braces on the test above, naming the four that were objects."""
+ payload = emit(Outcome())
+ for field in ("clients", "config", "strata"):
+ assert isinstance(payload[field], list), f"{field} must be a list of records"
+ assert isinstance(payload["quality"]["waste"], list)
+ # ...and the records have fixed field names, which is what makes the type
+ # stable no matter how the vocabulary grows.
+ assert isinstance(payload["shapes"]["by_content"], list)
+
+
+def test_trajectory_string_is_a_faithful_prefix_once_truncated():
+ """Past the cap the string must STOP, not keep growing its last run.
+
+ Letting a matching kind keep incrementing looks harmless and is not: once a
+ run has been dropped the remaining turns are no longer contiguous, so a run
+ of `s6` would claim six consecutive `s` turns where the session actually
+ did `s q s q s q`. Every run in the string has to be true.
+ """
+ runs_at_cap = []
+ for index in range(S.MAX_TRAJECTORY_RUNS):
+ runs_at_cap.append(
+ type(f"A{index}", (Outcome,), {"status_code": 500 if index % 2 else 200})()
+ )
+ # Then a tail that alternates between the last run's kind and another.
+ tail = []
+ for index in range(20):
+ tail.append(type(f"B{index}", (Outcome,), {"status_code": 200 if index % 2 else 429})())
+
+ payload = emit(*runs_at_cap, *tail)
+ kinds = payload["trajectory"]["kinds"]
+ assert payload["trajectory"]["kinds_truncated"] is True
+ runs = re.findall(r"([a-z])(\d+)", kinds)
+ assert len(runs) == S.MAX_TRAJECTORY_RUNS
+ # The prefix is exactly the pre-cap history: alternating runs of length 1.
+ assert all(int(n) == 1 for _, n in runs), f"a run grew after truncation: {kinds}"
+ # And the uncapped turn counter still accounts for every turn.
+ assert sum(payload["trajectory"]["turns"]) == payload["session"]["turns"]
+ assert payload["session"]["turns"] == S.MAX_TRAJECTORY_RUNS + 20
+
+
+def test_content_shape_cap_exceeds_its_vocabulary(beacon_on):
+ """The cap must not be able to bite in normal operation. It drops rows
+ first-come-wins, which would bias the table toward whatever a session
+ routed early — and that table is what a routing policy gets fitted on."""
+ from headroom.transforms.content_detector import ContentType
+ from headroom.transforms.content_router import CompressionStrategy
+
+ # +1 each for the "other" bucket a free string collapses to.
+ cross_product = (len(list(ContentType)) + 1) * (len(list(CompressionStrategy)) + 1)
+ assert S.MAX_SHAPE_ROWS >= cross_product, (
+ f"{cross_product} legitimate (content, strategy) pairs exist but the cap "
+ f"is {S.MAX_SHAPE_ROWS}"
+ )
+
+ for content in ContentType:
+ for strategy in CompressionStrategy:
+ S.record_content_shape(content.value, strategy.value, 100, 50)
+ rows = emit(Outcome())["shapes"]["by_content"]
+ assert len(rows) == len(list(ContentType)) * len(list(CompressionStrategy)), (
+ "a legitimate content/strategy combination was dropped"
+ )
+
+
+def test_the_client_budget_stays_under_every_deployed_receiver():
+ """`_MAX_WIRE_BYTES` is only useful while it is below what receivers accept.
+
+ The current Worker allows 256KB, but an install upgrades on its own schedule
+ and may be pointed at an older deployment whose cap is 64KB. Raising the
+ client budget past that would silently reintroduce the 413 this bounds.
+ """
+ worker = (Path(__file__).resolve().parents[1] / "deploy/beacon/worker.js").read_text()
+ match = re.search(r"const MAX_BODY_BYTES = (\d+) \* 1024;", worker)
+ assert match, "MAX_BODY_BYTES not found in worker.js"
+ assert S._MAX_WIRE_BYTES <= int(match.group(1)) * 1024
+ assert S._MAX_WIRE_BYTES <= 64 * 1024, (
+ f"the client budget is {S._MAX_WIRE_BYTES:,} B, over the 64KB cap of the "
+ "oldest Worker still in service"
+ )
+
+
+def test_the_worst_case_body_actually_sent_fits_the_oldest_receiver():
+ """The test above bounds what can be BUILT; this bounds what is SENT.
+
+ `_fit_to_wire` is the thing standing between the two, and uncompressed v2
+ crosses 64KB well before the shape caps are reached.
+ """
+ for index in range(S.MAX_SHAPE_ROWS + 20):
+ S.record_content_shape(f"content{index}", f"strategy{index}", 9000, 3000)
+ S.record_tool_shape(
+ type("Sig", (), {"field_count": 8, "max_depth": index % 10, "has_arrays": True})(),
+ 5000,
+ 1200,
+ )
+ outcomes = [
+ type(
+ f"O{index}",
+ (Outcome,),
+ {"status_code": 400 + (index % 99), "client": f"harness_{index % 70}"},
+ )()
+ for index in range(600)
+ ]
+ payload = emit(*outcomes)
+ saved = payload["tokens"]["saved"]
+ sent = S._fit_to_wire(payload, S.resource_attributes(), compress=False)
+ assert len(sent) <= 64 * 1024, f"sent {len(sent):,} B, over the 64KB receiver cap"
+ assert payload["tokens"]["saved"] == saved, "trimming cost a v1 counter"
diff --git a/tests/test_telemetry_warning.py b/tests/test_telemetry_warning.py
index c00b8bf1a..c8a70131a 100644
--- a/tests/test_telemetry_warning.py
+++ b/tests/test_telemetry_warning.py
@@ -75,7 +75,11 @@ class TestFormatTelemetryNotice:
monkeypatch.delenv(var, raising=False)
monkeypatch.delenv("HEADROOM_TELEMETRY_WARN", raising=False)
notice = format_telemetry_notice()
- assert "compression stats" in notice
+ # Substance, not the exact phrasing: what is sent (counters), what is
+ # not (prompts), and how to turn it off. The wording widened at schema
+ # v2 — "compression stats" stopped describing a payload that also
+ # carries cache behaviour, session shape and configuration.
+ assert "usage counters" in notice
assert "HEADROOM_BEACON=off" in notice
def test_beacon_notice_names_what_is_not_sent(self, monkeypatch):
@@ -233,7 +237,7 @@ class TestWrapCLITelemetryNotice:
_print_telemetry_notice()
captured = capsys.readouterr()
- assert "compression stats" in captured.out
+ assert "usage counters" in captured.out
assert "HEADROOM_BEACON=off" in captured.out
def test_print_notice_silent_when_telemetry_off(self, monkeypatch, capsys):