Python
"""OB-6: sending recordings to Langfuse or an OpenTelemetry endpoint (phase-2 design §10).
A recording is built in a real store with content planted everywhere a real session puts it:
the user's text, OpenWorker's per-turn context block, the system prompt and tool list, the
AI's answer and its thinking, a tool call's arguments and result (inside the next call's
history too), the auto-approver's input, answer and reason, a summary, the session title
call, a failed call's error text and cost, who answered a card, the machine's name, the
session title, and two secrets. Every planted text holds `PLANT`.
- Metadata only: no planted text in anything sent; a value planted in an exported field is
sent (the leak test of phase-2/lab/leak_test.py, as a repo test).
- Each switch alone lets out only its own content, and rules R1 to R5 hold.
- Sending: each step once per destination (marks survive a restart), retry and back-off,
a refused key, a refused span, against a fake OTLP endpoint on 127.0.0.1.
"""
from __future__ import annotations
import json
import threading
import time
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
import pytest
from coworker.observability import export as ex
from coworker.observability import export_spans as sp
from coworker.observability.capture import store_exchange
from coworker.observability.store import RecordingStore
VECTORS = Path(__file__).parent / "corpora" / "observability" / "golden-phase2" / "vectors" / "masking.json"
STORED_SECRET = "hunter2-correct-horse-PLANT"
KEY_SHAPED = "sk-PLANTabcdefghijklmnopqrstu"
T0 = 1_790_000_000_000
# -- a recording with content planted everywhere ------------------------------------------------
def _exchange(store: RecordingStore, request: dict, response: dict) -> dict:
return store_exchange({
"method": "POST", "url": "https://api.anthropic.com/v1/messages",
"request_headers": {"content-type": "application/json"},
"request_body": json.dumps(request).encode("utf-8"),
"status": 200, "response_headers": {}, "body": json.dumps(response),
}, store.put_piece)
def _reply(*blocks: dict, stop: str = "end_turn") -> dict:
return {"type": "message", "role": "assistant", "content": list(blocks), "stop_reason": stop}
def _usage(i: int = 100, o: int = 10) -> dict:
return {"reported": True, "input_fresh": i, "cache_read": 0, "cache_write": None, "output": o, "reasoning": None}
def _model(sid, tid, step_id, seq, at, purpose, exchange, *, parent=None, call_id=None, after_turn=False, status="ok",
model="anthropic:model-EXPORTED-zq81", **attrs):
a = {"model": model, "provider": "anthropic", "purpose": purpose, "stream": False,
"finish_reason": "end_turn", "tools_offered": 1, "usage": _usage(),
"provider_facts": {"cost_reported": 98765.4321, "served_model": "model-x"}}
a.update(attrs)
return {"step_id": step_id, "turn_id": tid, "session_id": sid, "parent_step_id": parent, "seq": seq,
"kind": "model", "name": "model", "started_at": at, "duration_ms": 400, "status": status,
"call_id": call_id, "after_turn": after_turn, "attrs": a,
"content": {"exchange": [exchange]} if exchange else {}}
def plant_turn(store: RecordingStore, sid: str = "sess-1", tid: str = "turn-1", at: int = T0,
*, long_text: str = "") -> None:
user = (f"USER-PLANT-u1 my key {KEY_SHAPED} and {STORED_SECRET} {long_text}"
"<system-context>CTX-PLANT-folders C:\\Users\\PLANT-home</system-context>")
system = "SYSTEM-PLANT-instructions"
tools = [{"name": "run_shell", "description": "TOOLDEF-PLANT", "input_schema": {"type": "object"}}]
m1 = _exchange(store, {"model": "model-x", "system": system, "tools": tools,
"messages": [{"role": "user", "content": user}]},
_reply({"type": "thinking", "thinking": "THINK-PLANT-reasoning"},
{"type": "text", "text": "ASSIST-PLANT-a1"},
{"type": "tool_use", "id": "c1", "name": "run_shell", "input": {"command": "ARGS-PLANT-cmd"}},
stop="tool_use"))
rv = _exchange(store, {"model": "model-x", "system": "You review actions.",
"messages": [{"role": "user", "content": "REVIEWER-IN-PLANT run_shell ARGS-PLANT-cmd"}]},
_reply({"type": "text", "text": "REVIEWER-OUT-PLANT allow"}))
m2 = _exchange(store, {"model": "model-x", "system": system, "tools": tools, "messages": [
{"role": "user", "content": user},
{"role": "assistant", "content": [{"type": "tool_use", "id": "c1", "name": "run_shell",
"input": {"command": "ARGS-PLANT-cmd"}}]},
{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "c1", "content": "RESULT-PLANT-out"}]}]},
_reply({"type": "text", "text": "ASSIST-PLANT-a2 final"}))
sm = _exchange(store, {"model": "model-x", "messages": [
{"role": "user", "content": "SUMMARY-IN-PLANT summarise RESULT-PLANT-out"}]},
_reply({"type": "text", "text": "SUMMARY-OUT-PLANT"}))
ti = _exchange(store, {"model": "model-x", "messages": [{"role": "user", "content": "TITLE-IN-PLANT"}]},
_reply({"type": "text", "text": "TITLE-OUT-PLANT"}))
failed = _exchange(store, {"model": "model-x", "messages": [{"role": "user", "content": user}]}, {})
store.write_turn({
"turn_id": tid, "session_id": sid, "seq": 1, "kind": "run", "trigger": "user", "started_at": at,
"duration_ms": 9000, "status": "completed", "approval_mode": "ask", "attended": True,
"stamp": {"machine": {"name": "PLANT-machine-name", "app_version": "9.9.9"}}, "schema_version": 1,
})
steps = [
{"step_id": f"{tid}-ctx", "kind": "context", "name": "context", "seq": 1, "started_at": at, "duration_ms": 1,
"status": "ok", "attrs": {"size_chars": 120, "block_hash": "abc"},
"content": {"agents_md": "AGENTS-PLANT"}},
_model(sid, tid, f"{tid}-m1", 2, at + 10, "main", m1),
{"step_id": f"{tid}-aa", "kind": "auto_approver", "name": "run_shell", "seq": 3, "started_at": at + 500,
"duration_ms": 300, "status": "ok", "call_id": "c1",
"attrs": {"mode": "live", "result": "allow", "reviewer": {"kind": "session_model", "model": "anthropic:model-x"}},
"content": {"reason": "REVIEWER-REASON-PLANT", "questions": [{"name": "QUESTIONS-PLANT"}]}},
_model(sid, tid, f"{tid}-rm", 4, at + 510, "auto_approver", rv, parent=f"{tid}-aa", call_id="c1"),
{"step_id": f"{tid}-w", "kind": "wait", "name": "approval", "seq": 5, "started_at": at + 900,
"duration_ms": 1000, "status": "ok",
"attrs": {"kind": "approval", "where": "session", "outcome": "approved", "attended": True, "for_call_id": "c1"},
"content": {"resolved_by": "PLANT-person-bob"}},
{"step_id": f"{tid}-t", "kind": "tool", "name": "run_shell", "seq": 6, "started_at": at + 2000,
"duration_ms": 500, "status": "ok", "call_id": "c1",
"attrs": {"risk_class": "exec", "exit_code": 0, "result_bytes": 16,
"decision": {"origin": "user", "path": [{"stage": "permission", "outcome": "asked"}]}},
"content": {"decision_details": "DECISION-PLANT", "blocked": "BLOCKED-PLANT"}},
_model(sid, tid, f"{tid}-m2", 7, at + 2600, "main", m2),
{"step_id": f"{tid}-sum", "kind": "summary", "name": "summary", "seq": 8, "started_at": at + 3200,
"duration_ms": 900, "status": "ok", "attrs": {"reason": "overflow", "tokens_after": 700}},
_model(sid, tid, f"{tid}-sm", 9, at + 3210, "summary", sm, parent=f"{tid}-sum"),
_model(sid, tid, f"{tid}-err", 10, at + 4200, "main", failed, status="error",
error={"type": "rate_limit", "text": "ERRTEXT-PLANT https://PLANT.example/v1"}, usage={"reported": False}),
{"step_id": f"{tid}-n", "kind": "notice", "name": "interrupted", "seq": 11, "started_at": at + 4700,
"duration_ms": 0, "status": "ok", "attrs": {}},
_model(sid, tid, f"{tid}-title", 12, at + 8000, "title", ti, after_turn=True),
]
for s in steps:
s.setdefault("turn_id", tid)
s.setdefault("session_id", sid)
store.write_step(s)
store.flush()
class Captured:
"""A `post` for the exporter that keeps what it was given and answers 200."""
def __init__(self, answers=None):
self.bodies: list[dict] = []
self.headers: list[dict] = []
self.answers = list(answers or [])
def __call__(self, url, body, headers, timeout):
self.bodies.append(json.loads(body))
self.headers.append(headers)
return self.answers.pop(0) if self.answers else (200, "")
def text(self) -> str:
return json.dumps(self.bodies, ensure_ascii=False)
def spans(self) -> list[dict]:
return [s for b in self.bodies for r in b["resourceSpans"] for ss in r["scopeSpans"] for s in ss["spans"]]
def _attr(span: dict, key: str):
for a in span["attributes"]:
if a["key"] == key:
v = a["value"]
return next(iter(v.values()))
return None
def export_once(tmp_path, include: dict, *, dest=None, long_text: str = "") -> Captured:
store = RecordingStore(tmp_path / "recordings.db", background=False)
plant_turn(store, long_text=long_text)
cap = Captured()
cfg = ex.ExportConfig(destination=dest or ex.Destination(kind="langfuse", host="http://127.0.0.1:9"),
headers={"Authorization": "Basic x"}, include=include, since_ms=0)
exporter = ex.Exporter(store, cfg, post=cap, secret_values=lambda: [STORED_SECRET],
session_title=lambda _sid: "TITLE-PLANT-session")
exporter.flush_once()
store.close()
return cap
OFF = {"messages": False, "tool_io": False, "context": False, "names": False}
def on(*keys: str) -> dict:
return {**OFF, **{k: True for k in keys}}
# -- 2. metadata only carries no content --------------------------------------------------------------
@pytest.mark.parametrize("dest", [ex.Destination(kind="langfuse", host="http://127.0.0.1:9"),
ex.Destination(kind="otlp", url="http://127.0.0.1:9/v1/traces")])
def test_metadata_only_export_carries_no_content(tmp_path, dest):
cap = export_once(tmp_path, OFF, dest=dest)
sent = cap.text()
assert len(cap.spans()) == 12 # the turn and 11 steps; the notice is an event on the turn
# The positive control: a value planted in an exported field is found by the same search.
assert "model-EXPORTED-zq81" in sent
# Nothing planted in content is.
assert "PLANT" not in sent
assert "98765" not in sent # provider-reported cost (R5)
assert "C:\\\\Users" not in sent and "Users" not in sent
# Metadata is there: names, kinds, counts, the verdict, the notice.
spans = cap.spans()
names = {s["name"] for s in spans}
assert {"invoke_agent coworker", "openworker.context.build", "chat model-EXPORTED-zq81",
"openworker.policy.review run_shell", "openworker.approval.wait", "execute_tool run_shell",
"openworker.compaction"} <= names
review = next(s for s in spans if s["name"] == "openworker.policy.review run_shell")
assert _attr(review, "openworker.reviewer.result") == "allow"
root = next(s for s in spans if s["name"] == "invoke_agent coworker")
assert [e["name"] for e in root["events"]] == ["openworker.notice.interrupted"]
failed = next(s for s in spans if _attr(s, "openworker.step.id") == "turn-1-err")
assert _attr(failed, "error.type") == "rate_limit" and failed["status"]["code"] == 2
# -- 3. each switch lets out only its own content --------------------------------------------------------
ALL_PLANTS = ["USER-PLANT", "ASSIST-PLANT", "CTX-PLANT", "SYSTEM-PLANT", "TOOLDEF-PLANT", "ARGS-PLANT",
"RESULT-PLANT", "REVIEWER-IN-PLANT", "REVIEWER-OUT-PLANT", "REVIEWER-REASON-PLANT",
"SUMMARY-IN-PLANT", "SUMMARY-OUT-PLANT", "TITLE-IN-PLANT", "TITLE-OUT-PLANT",
"TITLE-PLANT-session", "PLANT-person-bob", "PLANT-machine-name"]
NEVER = ["THINK-PLANT", "ERRTEXT-PLANT", "98765", "QUESTIONS-PLANT", "DECISION-PLANT", "AGENTS-PLANT",
"BLOCKED-PLANT", STORED_SECRET, KEY_SHAPED]
CASES = {
# R1, R2, R3: the user's and the AI's own text, the title call; no tool parts, no context
# block, no reviewer or summary text.
"messages": (on("messages"), {"USER-PLANT", "ASSIST-PLANT", "TITLE-IN-PLANT", "TITLE-OUT-PLANT"}),
# Tool spans only: the reviewer's input and reason need the messages switch too (R3).
"tool_io": (on("tool_io"), {"ARGS-PLANT", "RESULT-PLANT"}),
"context": (on("context"), {"SYSTEM-PLANT", "TOOLDEF-PLANT"}),
"names": (on("names"), {"TITLE-PLANT-session", "PLANT-person-bob", "PLANT-machine-name"}),
"messages+tool_io": (on("messages", "tool_io"),
{"USER-PLANT", "ASSIST-PLANT", "ARGS-PLANT", "RESULT-PLANT", "REVIEWER-IN-PLANT",
"REVIEWER-OUT-PLANT", "REVIEWER-REASON-PLANT", "SUMMARY-IN-PLANT", "SUMMARY-OUT-PLANT",
"TITLE-IN-PLANT", "TITLE-OUT-PLANT"}),
"messages+context": (on("messages", "context"),
{"USER-PLANT", "ASSIST-PLANT", "CTX-PLANT", "SYSTEM-PLANT", "TOOLDEF-PLANT",
"TITLE-IN-PLANT", "TITLE-OUT-PLANT"}),
"everything": (on("messages", "tool_io", "context", "names"), set(ALL_PLANTS)),
}
@pytest.mark.parametrize("case", list(CASES))
def test_each_switch_lets_out_only_its_content(tmp_path, case):
include, allowed = CASES[case]
sent = export_once(tmp_path, include).text()
for plant in ALL_PLANTS:
assert (plant in sent) == (plant in allowed), f"{case}: {plant}"
for plant in NEVER: # R5 and R4, whatever the switches say
assert plant not in sent, f"{case}: {plant}"
def test_tool_parts_inside_messages_are_masked_unless_the_tool_switch_is_on(tmp_path):
cap = export_once(tmp_path, on("messages"))
main = [s for s in cap.spans() if _attr(s, "openworker.step.id") == "turn-1-m2"][0]
history = json.loads(_attr(main, "gen_ai.input.messages"))
parts = [p for m in history for p in m["parts"]]
call = next(p for p in parts if p["type"] == "tool_call")
result = next(p for p in parts if p["type"] == "tool_call_response")
# R1: the part, its id and its tool name stay; its content does not.
assert (call["id"], call["name"], call["arguments"]) == ("c1", "run_shell", sp.TOOL_ARGS_OUT)
assert (result["id"], result["response"]) == ("c1", sp.TOOL_RESULT_OUT)
# R2: the context block is cut out of the user's text, the rest kept.
user = next(p["content"] for m in history if m["role"] == "user" for p in m["parts"] if p["type"] == "text")
assert user.startswith("USER-PLANT-u1") and user.endswith(sp.CONTEXT_OUT)
# R4: the stored secret and the key-shaped one are masked.
assert user.count(sp.SECRET) == 2
# Reasoning keeps its marker, never its text (R5).
reply = json.loads(_attr([s for s in cap.spans() if _attr(s, "openworker.step.id") == "turn-1-m1"][0],
"gen_ai.output.messages"))
assert {"type": "reasoning", "content": sp.REASONING} in reply[0]["parts"]
# The turn's own input and output (Langfuse).
root = cap.spans()[0]
assert _attr(root, "langfuse.trace.input").endswith(sp.CONTEXT_OUT)
assert _attr(root, "langfuse.trace.output") == "ASSIST-PLANT-a2 final"
def test_tool_switch_with_messages_lets_tool_parts_into_messages(tmp_path):
cap = export_once(tmp_path, on("messages", "tool_io"))
main = [s for s in cap.spans() if _attr(s, "openworker.step.id") == "turn-1-m2"][0]
history = _attr(main, "gen_ai.input.messages")
assert "ARGS-PLANT-cmd" in history and "RESULT-PLANT-out" in history
tool = [s for s in cap.spans() if s["name"] == "execute_tool run_shell"][0]
assert json.loads(_attr(tool, "gen_ai.tool.call.arguments")) == {"command": "ARGS-PLANT-cmd"}
assert _attr(tool, "gen_ai.tool.call.result") == "RESULT-PLANT-out"
# gen_ai.tool.call.id only on the tool's own span (V-22).
assert [s["name"] for s in cap.spans() if _attr(s, "gen_ai.tool.call.id")] == ["execute_tool run_shell"]
def test_exported_text_is_cut_at_8_kb(tmp_path):
cap = export_once(tmp_path, on("messages"), long_text="x" * 20_000)
main = [s for s in cap.spans() if _attr(s, "openworker.step.id") == "turn-1-m1"][0]
user = json.loads(_attr(main, "gen_ai.input.messages"))[0]["parts"][0]["content"]
assert user.endswith(" more bytes]") and "[cut: " in user
kept = user[: user.index("[cut: ")]
assert len(kept.encode("utf-8")) == sp.CUT_BYTES
def test_masking_and_cut_match_the_shared_vectors():
vectors = json.loads(VECTORS.read_text(encoding="utf-8"))
for v in vectors["mask"]:
assert sp.Masker(v["secrets"]).mask(v["text"]) == v["expected"], v["text"]
for v in vectors["cut"]:
text = "a" * 8191 + "é" + "b" * 10
assert len(text.encode("utf-8")) == v["text_bytes"]
out = sp.cut(text)
kept = out[: out.index("[cut:")]
assert len(kept.encode("utf-8")) == v["expected_kept_bytes"]
assert out.endswith(v["expected_tail"])
# The patterns are the ones in attributes.yaml export.content_switches.masking.
assert sp.CUT_BYTES == 8192 and len(sp.MASK_PATTERNS) == 7
def test_credential_looking_environment_variables_are_masked():
env = {"MY_API_KEY": "PLANT-envkey-123", "SHORT_TOKEN": "abc", "HOME": "PLANT-not-a-secret"}
m = sp.Masker(sp.env_secret_values(env))
assert m.mask("a PLANT-envkey-123 b abc PLANT-not-a-secret") == "a [secret masked] b abc PLANT-not-a-secret"
def test_token_numbers_follow_the_verified_rules(tmp_path):
cap = export_once(tmp_path, OFF)
spans = cap.spans()
m1 = [s for s in spans if _attr(s, "openworker.step.id") == "turn-1-m1"][0]
assert _attr(m1, "gen_ai.usage.input_tokens") == "100"
assert _attr(m1, "gen_ai.usage.cache_read.input_tokens") == "0"
assert _attr(m1, "gen_ai.usage.cache_creation.input_tokens") is None # not reported: left out
assert json.loads(_attr(m1, "langfuse.observation.usage_details")) == {"input": 100, "input_cache_read": 0, "output": 10}
root = spans[0]
assert _attr(root, "gen_ai.usage.input_tokens") is None # never on a parent span
assert _attr(root, "openworker.usage.failed_calls") == "1"
assert _attr(root, "langfuse.trace.tags") == {"values": [{"stringValue": "usage-incomplete"}]}
err = [s for s in spans if _attr(s, "openworker.step.id") == "turn-1-err"][0]
assert _attr(err, "langfuse.observation.level") == "ERROR"
assert _attr(err, "langfuse.observation.status_message") == "rate_limit; usage unknown"
# One trace per turn, children under their parents.
assert len({s["traceId"] for s in spans}) == 1
rm = [s for s in spans if _attr(s, "openworker.step.id") == "turn-1-rm"][0]
aa = [s for s in spans if _attr(s, "openworker.step.id") == "turn-1-aa"][0]
assert rm["parentSpanId"] == aa["spanId"] and aa["parentSpanId"] == root["spanId"]
resource = cap.bodies[0]["resourceSpans"][0]["resource"]["attributes"]
assert {"key": "openworker.export.schema", "value": {"stringValue": "openworker/1.3"}} in resource
assert not any(a["key"] == "host.name" for a in resource) # the machine name: names switch only
# -- 4. each step once; retry; refused key; refused span ------------------------------------------------
class FakeOtlp:
"""A local OTLP/HTTP endpoint. `answers` is a list of status codes for the next posts;
`refuse` is a set of span ids refused (400) whenever they are in a post."""
def __init__(self):
self.answers: list[int] = []
self.refuse: set[str] = set()
self.received: list[dict] = []
self.auth: list[str] = []
outer = self
class Handler(BaseHTTPRequestHandler):
def do_POST(self): # noqa: N802
body = json.loads(self.rfile.read(int(self.headers["Content-Length"])))
outer.auth.append(self.headers.get("Authorization") or "")
ids = [s["spanId"] for r in body["resourceSpans"] for ss in r["scopeSpans"] for s in ss["spans"]]
code = outer.answers.pop(0) if outer.answers else 200
if code == 200 and outer.refuse & set(ids):
code = 400
if code == 200:
outer.received.append(body)
self.send_response(code)
self.end_headers()
self.wfile.write(b'{"message":"refused here"}' if code != 200 else b"{}")
def log_message(self, *a):
pass
self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
self.url = f"http://127.0.0.1:{self.server.server_address[1]}/v1/traces"
threading.Thread(target=self.server.serve_forever, daemon=True).start()
def span_ids(self) -> list[str]:
return [s["spanId"] for b in self.received for r in b["resourceSpans"] for ss in r["scopeSpans"] for s in ss["spans"]]
def close(self):
self.server.shutdown()
@pytest.fixture
def otlp():
f = FakeOtlp()
yield f
f.close()
def _cfg(url: str, since: int = 0) -> ex.ExportConfig:
return ex.ExportConfig(destination=ex.Destination(kind="otlp", url=url), headers={"Authorization": "Bearer t"},
include=OFF, since_ms=since)
def test_each_step_is_sent_once_and_the_marks_survive_a_restart(tmp_path, otlp):
db = tmp_path / "recordings.db"
store = RecordingStore(db, background=False)
plant_turn(store, tid="turn-1")
plant_turn(store, tid="turn-2", at=T0 + 20_000)
exporter = ex.Exporter(store, _cfg(otlp.url))
assert exporter.flush_once() == 24
assert exporter.flush_once() == 0 # nothing new
store.close()
# A restart: a new store on the same file, a new exporter.
store = RecordingStore(db, background=False)
assert ex.Exporter(store, _cfg(otlp.url)).flush_once() == 0
# The status's "last sent" survives the restart: it comes from the marks.
assert store.export_counts("otlp:" + otlp.url)["last_at"]
ids = otlp.span_ids()
assert len(ids) == len(set(ids)) == 24
# A step that ends after its turn was sent (a late title call) is sent alone, once.
store.write_step({"step_id": "turn-1-late", "turn_id": "turn-1", "session_id": "sess-1", "seq": 13,
"kind": "model", "name": "model", "started_at": T0 + 9500, "duration_ms": 100, "status": "ok",
"after_turn": True, "attrs": {"model": "anthropic:model-x", "purpose": "title", "usage": _usage()}})
store.flush()
assert ex.Exporter(store, _cfg(otlp.url)).flush_once() == 1
assert len(otlp.span_ids()) == 25
assert store.export_counts("otlp:" + otlp.url)["sent"] == 2 * 12 + 2 * 1 + 1 # notices ride with their turn
store.close()
def test_a_new_destination_starts_from_turns_that_end_after_the_change(tmp_path, otlp):
store = RecordingStore(tmp_path / "recordings.db", background=False)
plant_turn(store, tid="old", at=T0)
plant_turn(store, tid="new", at=T0 + 60_000)
ex.Exporter(store, _cfg(otlp.url, since=T0 + 30_000)).flush_once()
assert {s["traceId"] for b in otlp.received for r in b["resourceSpans"] for ss in r["scopeSpans"]
for s in ss["spans"]} == {sp.trace_id("new")}
store.close()
def _wait_for(cond, timeout=10.0):
end = time.time() + timeout
while time.time() < end:
if cond():
return True
time.sleep(0.02)
return False
def test_retry_with_back_off_then_sent(tmp_path, otlp):
store = RecordingStore(tmp_path / "recordings.db", background=False)
plant_turn(store)
otlp.answers = [503, 503, 200]
exporter = ex.Exporter(store, _cfg(otlp.url), backoff_start=0.05, flush_seconds=0.05)
seen: list[tuple[str, str]] = []
exporter.start()
try:
assert _wait_for(lambda: seen.append((exporter.state, exporter.last_error)) or len(otlp.span_ids()) == 12)
finally:
exporter.stop()
assert ("retrying", "HTTP 503: {\"message\":\"refused here\"}") in seen
assert exporter.state == "sending" and exporter.last_error == ""
store.close()
def test_a_network_error_is_retried(tmp_path):
store = RecordingStore(tmp_path / "recordings.db", background=False)
plant_turn(store)
exporter = ex.Exporter(store, _cfg("http://127.0.0.1:1/v1/traces"), backoff_start=0.05)
with pytest.raises(ex._Retry):
exporter.flush_once()
assert store.export_marked("otlp:http://127.0.0.1:1/v1/traces", "turn-1") == set()
store.close()
def test_a_refused_key_stops_sending_until_the_settings_change(tmp_path, otlp):
store = RecordingStore(tmp_path / "recordings.db", background=False)
plant_turn(store)
otlp.answers = [401]
exporter = ex.Exporter(store, _cfg(otlp.url), backoff_start=0.05, flush_seconds=0.05)
exporter.start()
try:
assert _wait_for(lambda: exporter.state == "failed")
time.sleep(0.2)
assert otlp.span_ids() == [] # it stopped: no second try with the same key
assert exporter.last_error == "The destination refused the key"
assert store.export_counts("otlp:" + otlp.url)["failed"] == 1
finally:
exporter.stop()
store.close()
def test_a_span_refused_on_its_own_is_dropped_and_the_rest_sent(tmp_path, otlp):
store = RecordingStore(tmp_path / "recordings.db", background=False)
plant_turn(store)
otlp.refuse = {sp.step_span_id("turn-1-t")}
ex.Exporter(store, _cfg(otlp.url)).flush_once()
ids = otlp.span_ids()
assert len(ids) == 11 and sp.step_span_id("turn-1-t") not in ids
assert store.export_counts("otlp:" + otlp.url)["dropped"] == 1
store.close()
def test_a_turn_deleted_before_it_was_sent_counts_as_dropped(tmp_path):
store = RecordingStore(tmp_path / "recordings.db", background=False, retention_days=7)
plant_turn(store, at=T0)
store.export_destination = ("otlp:http://x/v1/traces", 0)
store.sweep(now_ms=T0 + 8 * 24 * 3600 * 1000)
assert store.export_counts("otlp:http://x/v1/traces")["dropped"] == 1
store.close()
def test_batches_hold_at_most_the_batch_size(tmp_path):
store = RecordingStore(tmp_path / "recordings.db", background=False)
for i in range(3):
plant_turn(store, tid=f"t{i}", at=T0 + i * 20_000)
cap = Captured()
ex.Exporter(store, _cfg("http://127.0.0.1:9/v1/traces"), post=cap, batch_spans=5).flush_once()
sizes = [len([s for r in b["resourceSpans"] for ss in r["scopeSpans"] for s in ss["spans"]]) for b in cap.bodies]
assert max(sizes) <= 5 and sum(sizes) == 36
store.close()
# -- settings, keys and the test recording ------------------------------------------------------------
class FakeSecrets:
def __init__(self):
self.data: dict = {}
def get(self, profile):
return self.data.get(profile)
def put(self, profile, value):
self.data[profile] = json.loads(json.dumps(value))
def delete(self, profile):
return self.data.pop(profile, None) is not None
def _read(self):
return self.data
def test_settings_keep_keys_out_of_the_settings_file_and_answers(tmp_path):
secrets = FakeSecrets()
store = RecordingStore(tmp_path / "recordings.db", background=False)
obs = ex.ObservabilityExport(tmp_path, secrets, store, post=Captured())
try:
assert obs.exporter is None and obs.status()["state"] == "off"
code, answer = obs.put_settings({
"destination": {"kind": "langfuse", "host": "https://langfuse.example.com"},
"secrets": {"public_key": "pk-lf-PLANT-public-3a91", "secret_key": "sk-lf-PLANT-secret-key"},
"include": {"messages": True}})
assert code == 200 and answer["ok"]
assert answer["secrets"] == {"public_key": "pk-lf-••••••••3a91", "secret_key": "sk-lf-••••••••"}
assert "PLANT" not in json.dumps(answer) and "PLANT" not in json.dumps(obs.get_settings())
assert "PLANT" not in (tmp_path / ex.SETTINGS_FILE).read_text(encoding="utf-8")
assert secrets.data[ex.PROFILE]["user"]["secret_key"] == "sk-lf-PLANT-secret-key"
assert obs.exporter is not None and obs.exporter.config.destination.endpoint == \
"https://langfuse.example.com/api/public/otel/v1/traces"
# Off again: the exporter goes away, its thread with it.
obs.put_settings({"destination": {"kind": "off"}})
assert obs.exporter is None
assert not any(t.name == "observability-export" for t in threading.enumerate())
# Bad input is refused, not saved.
assert obs.put_settings({"destination": {"kind": "otlp", "url": "ftp://x"}})[0] == 400
assert obs.put_settings({"retention_days": 12})[0] == 400
finally:
obs.close()
store.close()
def test_the_test_recording_reports_a_refused_key(tmp_path, otlp):
store = RecordingStore(tmp_path / "recordings.db", background=False)
obs = ex.ObservabilityExport(tmp_path, FakeSecrets(), store)
try:
obs.put_settings({"destination": {"kind": "otlp", "url": otlp.url, "header_names": ["Authorization"]},
"secrets": {"headers": {"Authorization": "Bearer PLANT-token"}}})
otlp.answers = [200]
ok = obs.send_test()
assert ok["ok"] and ok["text"] == "Sent." and ok["status"] == 200
assert otlp.auth[-1] == "Bearer PLANT-token"
[span_names] = [[s["name"] for r in b["resourceSpans"] for ss in r["scopeSpans"] for s in ss["spans"]]
for b in otlp.received[-1:]]
assert span_names == ["invoke_agent coworker", "chat test-model", "execute_tool test_tool"]
otlp.answers = [401]
refused = obs.send_test()
assert refused == {"ok": False, "status": 401, "text": "The destination refused the key", "ms": refused["ms"]}
otlp.answers = [500]
assert obs.send_test()["text"].startswith("HTTP 500")
finally:
obs.close()
store.close()
POLICY = {"org_id": "org_1", "org_name": "Acme Corp", "version": 3, "observability_export": {
"enabled": True, "name": "Security team default", "set_at": "2026-10-12",
"destination": {"kind": "langfuse", "host": "https://langfuse.acme.internal",
"public_key": "pk-lf-POLICY-PLANT-0001", "secret_key": "sk-lf-POLICY-PLANT"},
"include": {"messages": True, "names": True}}}
def test_a_company_policy_manages_the_page(tmp_path):
secrets = FakeSecrets()
store = RecordingStore(tmp_path / "recordings.db", background=False)
obs = ex.ObservabilityExport(tmp_path, secrets, store, post=Captured())
try:
stripped = ex.take_policy_secrets(POLICY, secrets)
assert "PLANT" not in json.dumps(stripped) # what policy.json will hold
assert secrets.data[ex.PROFILE]["policy"]["secret_key"] == "sk-lf-POLICY-PLANT"
obs.apply_policy(stripped)
got = obs.get_settings()
assert got["managed"] == {"by": "Acme Corp", "name": "Security team default", "at": "2026-10-12"}
assert got["destination"] == {"kind": "langfuse", "host": "https://langfuse.acme.internal"}
assert got["include"] == {"messages": True, "tool_io": False, "context": False, "names": True}
assert got["notice"] == "Acme Corp receives this session's messages and names (Langfuse, langfuse.acme.internal)."
assert "PLANT" not in json.dumps(got)
assert obs.put_settings({"include": {"tool_io": True}})[0] == 409
assert obs.put_settings({"destination": {"kind": "off"}})[0] == 409
assert obs.put_settings({"retention_days": 7})[0] == 200 # on this machine stays the person's
assert obs.exporter is not None and obs.exporter.config.managed
assert obs.status()["state"] in ("managed", "sending", "retrying")
# Anything malformed reads as not managed.
for bad in ({"observability_export": {"enabled": "yes"}},
{"observability_export": {"enabled": True, "destination": {"kind": "smtp"}}},
{"observability_export": []}):
assert ex.managed_export(bad) is None
# The policy taken away: the person's own settings are back, the policy's keys gone.
obs.apply_policy(ex.take_policy_secrets({"org_id": "org_1"}, secrets))
assert obs.get_settings()["managed"] is None and "policy" not in (secrets.data.get(ex.PROFILE) or {})
finally:
obs.close()
store.close()