Skip to content
Python
14.5 KB
Raw Blame
"""Observability E2, E22, E3: every model call is a model step, with its exact exchange and
correct token counts.

- E3: the shared usage vectors (`golden-phase1/vectors/usage.json`, recording format 1.0
  section 5) go through each provider's own usage code and come out as the step's `usage`.
- E2: a real engine turn records one model step per call, purpose `main`, pointing at the
  reply message it produced; untagged and proxy calls are told apart.
- E22: a call to a local fake server is captured below the SDK: the request rebuilds
  exactly from its pieces, no credential is stored, and `provider_facts` are read.
"""

from __future__ import annotations

import asyncio
import json
import threading
from http.server import BaseHTTPRequestHandler, HTTPServer
from pathlib import Path
from types import SimpleNamespace

import aisuite as ai
import pytest

from coworker.engine import TurnEngine
from coworker.observability import Recorder
from coworker.observability import capture
from coworker.observability.engine_hooks import TurnRecording
from coworker.observability.provider_view import RecordingProvider, usage_attrs
from coworker.permissions import PermissionEngine
from coworker.providers import AssistantTurn, ModelCapabilities, ProviderClient, ToolCall
from coworker.providers import anthropic_provider, bedrock_provider, gemini_provider
from coworker.providers import openai_provider, openai_responses
from coworker.providers.base import TokenUsage
from coworker.providers.ollama_context import to_openai_completion
from coworker.tools import ToolRegistry

VECTORS = json.loads(
    (Path(__file__).parent / "corpora/observability/golden-phase1/vectors/usage.json").read_text("utf-8")
)


def _ns(v):
    if isinstance(v, dict):
        return SimpleNamespace(**{k: _ns(x) for k, x in v.items()})
    return v


def _provider_usage(family, raw):
    if family == "anthropic":
        return anthropic_provider._usage_from(_ns(raw))
    if family == "openai_chat":
        return openai_provider._usage_from(_ns(raw))
    if family == "openai_responses":
        return openai_responses._usage_from(_ns(raw))
    if family == "gemini":
        return gemini_provider._usage_from(_ns(raw))
    if family == "bedrock_converse":
        return bedrock_provider._usage_from(raw)
    if family == "ollama":
        # The app turns Ollama's native reply into an OpenAI-shaped one (ollama_context.py).
        return openai_provider._usage_from(_ns(to_openai_completion({"message": {}, **raw})["usage"]))
    raise AssertionError(family)


# The decision reviewers call their own service directly; their usage comes with E23.
PROVIDER_VECTORS = [v for v in VECTORS if v["family"] not in ("openai_decisions", "typesafe_jev")]


@pytest.mark.parametrize("vector", PROVIDER_VECTORS, ids=[v["id"] for v in PROVIDER_VECTORS])
def test_usage_vectors(vector):
    got = usage_attrs(_provider_usage(vector["family"], vector["raw"]))
    assert got == {"reported": True, **vector["expected"]}


def test_a_call_with_no_usage_is_not_reported():
    assert usage_attrs(None) == {"reported": False}
    assert usage_attrs(TokenUsage(input=5, output=1, cache_write_reported=False))["cache_write"] is None


def test_anthropic_stream_totals_come_from_the_final_event():
    start = _ns({"input_tokens": 5, "output_tokens": 1, "cache_read_input_tokens": 5194,
                 "cache_creation_input_tokens": 130})
    final = _ns({"output_tokens": 23, "output_tokens_details": {"thinking_tokens": 0}})
    u = anthropic_provider._merge_final_usage(anthropic_provider._usage_from(start), final)
    assert usage_attrs(u) == {"reported": True, "input_fresh": 5, "cache_read": 5194,
                              "cache_write": 130, "output": 23, "reasoning": 0}


# -- E2: model steps from a real engine turn -------------------------------------------------

class ScriptedProvider(ProviderClient):
    def __init__(self, turns):
        self._turns = list(turns)

    def complete(self, *, model, messages, tools=None, **settings):
        turn = self._turns.pop(0)
        if isinstance(turn, Exception):
            raise turn
        return turn

    def capabilities(self, model):
        return ModelCapabilities()


def _engine(tmp_path, turns):
    registry = ToolRegistry()
    registry.register_all(ai.toolkits.files(root=str(tmp_path), allow_write=True))
    engine = TurnEngine(
        provider=ScriptedProvider(turns),
        registry=registry,
        permissions=PermissionEngine(workspace_root=tmp_path),
        model="openai:model-a",
    )
    engine.audit_context = {"session_id": "sess_1"}
    rec = Recorder()
    engine.recorder = rec
    return engine, rec


def _run(engine, text):
    async def go():
        return [ev async for ev in engine.run(text, trigger="user")]

    return asyncio.run(go())


def test_every_model_call_is_a_main_model_step(tmp_path):
    (tmp_path / "a.txt").write_text("hello", encoding="utf-8")
    engine, rec = _engine(tmp_path, [
        AssistantTurn(tool_calls=[ToolCall(id="c1", name="read_file", arguments={"path": "a.txt"})],
                      finish_reason="tool_calls",
                      usage=TokenUsage(input=100, output=10, cache_read=50, cache_write_reported=False)),
        AssistantTurn(text="it says hello", finish_reason="stop", usage=None),
    ])
    _run(engine, "read it")
    d = rec.as_dict()
    models = [s for s in d["steps"] if s["kind"] == "model"]
    assert [s["attrs"]["purpose"] for s in models] == ["main", "main"]
    assistants = [m for m in engine.messages if m["role"] == "assistant"]
    assert [s["message_id"] for s in models] == [m["mid"] for m in assistants]
    first, second = models
    assert first["attrs"]["finish_reason"] == "tool_calls"
    assert first["attrs"]["usage"] == {"reported": True, "input_fresh": 100, "cache_read": 50,
                                       "cache_write": None, "output": 10, "reasoning": None}
    assert second["attrs"]["usage"] == {"reported": False}
    assert first["attrs"]["model"] == "openai:model-a" and first["attrs"]["provider"] == "openai"
    assert first["attrs"]["tools_offered"] > 0 and first["attrs"]["tools_hash"]
    assert all(s["turn_id"] == d["turns"][0]["turn_id"] for s in models)
    # canonical order: the context taken at the first model call, then model, tool, model
    assert [s["kind"] for s in d["steps"]] == ["context", "model", "tool", "model"]


def test_a_failed_model_call_is_an_error_step(tmp_path):
    engine, rec = _engine(tmp_path, [RuntimeError("upstream 529 overloaded")])
    _run(engine, "hi")
    [step] = [s for s in rec.as_dict()["steps"] if s["kind"] == "model"]
    assert step["status"] == "error"
    assert step["attrs"]["error"]["type"] == "overloaded"


def test_untagged_and_proxy_calls_are_told_apart():
    rec = Recorder()
    base = RecordingProvider(ScriptedProvider([AssistantTurn(text="a"), AssistantTurn(text="b")]), recorder=rec)
    base.complete(model="openai:model-a", messages=[{"role": "user", "content": "x"}])
    base.tagged(purpose="proxy", turn=None).complete(model="openai:model-a", messages=[])
    purposes = sorted(s["attrs"]["purpose"] for s in rec.as_dict()["steps"])
    assert purposes == ["proxy", "unattributed"]
    assert all(s["turn_id"] in ("", None) for s in rec.as_dict()["steps"])


def test_a_title_tagged_with_an_ended_turn_is_after_turn(tmp_path):
    engine, rec = _engine(tmp_path, [AssistantTurn(text="done"), AssistantTurn(text="Title")])
    _run(engine, "hello")
    turn_id = engine._obs.turn_id
    engine.provider.tagged(purpose="title", turn=turn_id).complete(model=engine.model, messages=[])
    title = next(s for s in rec.as_dict()["steps"] if s["attrs"].get("purpose") == "title")
    assert title["turn_id"] == turn_id and title["after_turn"] is True


# -- E22: the exact exchange, captured below the SDK ------------------------------------------

_REPLY = {
    "id": "chatcmpl-1", "object": "chat.completion", "created": 1, "model": "model-a-2026",
    "choices": [{"index": 0, "message": {"role": "assistant", "content": "hi there"},
                 "finish_reason": "stop"}],
    "usage": {"prompt_tokens": 30, "completion_tokens": 3, "total_tokens": 33,
              "prompt_tokens_details": {"cached_tokens": 20}},
}


class _Handler(BaseHTTPRequestHandler):
    seen: list = []

    def do_POST(self):  # noqa: N802
        body = self.rfile.read(int(self.headers.get("content-length") or 0))
        _Handler.seen.append((dict(self.headers), body))
        data = json.dumps(_REPLY).encode()
        self.send_response(200)
        self.send_header("content-type", "application/json")
        self.send_header("x-request-id", "req_abc")
        self.send_header("content-length", str(len(data)))
        self.end_headers()
        self.wfile.write(data)

    def log_message(self, *a):
        pass


@pytest.fixture
def fake_server():
    server = HTTPServer(("127.0.0.1", 0), _Handler)
    t = threading.Thread(target=server.serve_forever, daemon=True)
    t.start()
    yield f"http://127.0.0.1:{server.server_address[1]}/v1"
    server.shutdown()


def test_the_exact_exchange_is_kept_and_no_key_is_stored(fake_server):
    rec = Recorder()
    sink = TurnRecording(rec, lambda: "sess_x")
    inner = openai_provider.OpenAIProvider(api_key="sk-test-not-real", base_url=fake_server)
    view = RecordingProvider(inner).tagged(purpose="main", sink=sink)
    messages = [{"role": "system", "content": "be brief"}, {"role": "user", "content": "say hi"}]
    turn = view.complete(model="model-a", messages=messages)
    assert turn.text == "hi there"

    [step] = rec.as_dict()["steps"]
    [exchange] = step["content"]["exchange"]
    sent_headers, sent_body = _Handler.seen[-1]
    # Rebuilds exactly, and the stored hash is the hash of the bytes sent.
    assert capture.rebuild_request(exchange, rec.get_piece) == json.loads(sent_body)
    import hashlib
    assert exchange["request"]["body_sha256"] == hashlib.sha256(sent_body).hexdigest()
    assert json.loads(rec.get_piece(exchange["response"]["body_blob"])) == _REPLY
    # No credential anywhere in what is stored.
    stored = json.dumps(step)
    assert "sk-test-not-real" not in stored
    assert "authorization" not in {k.lower() for k in exchange["request"]["headers"]}
    # Metadata without content; facts with fixed names.
    meta = step["attrs"]["provider_meta"][0]
    assert meta["wire"] == "openai_chat" and "messages" not in meta["request"]
    assert "hi there" not in json.dumps(step["attrs"])
    facts = step["attrs"]["provider_facts"]
    assert facts["served_model"] == "model-a-2026"
    assert facts["provider_request_id"] == "req_abc"
    assert facts["provider_total"] == 33
    assert step["attrs"]["usage"] == {"reported": True, "input_fresh": 10, "cache_read": 20,
                                      "cache_write": None, "output": 3, "reasoning": None}


def test_capture_is_off_outside_a_recorded_call(fake_server):
    inner = openai_provider.OpenAIProvider(api_key="sk-test-not-real", base_url=fake_server)
    capture.install()
    assert getattr(capture._active, "log", None) is None
    inner.complete(model="model-a", messages=[{"role": "user", "content": "x"}])
    assert getattr(capture._active, "log", None) is None


DECISION_VECTORS = [v for v in VECTORS if v["family"] in ("openai_decisions", "typesafe_jev")]


@pytest.mark.parametrize("vector", DECISION_VECTORS, ids=[v["id"] for v in DECISION_VECTORS])
def test_decision_reviewer_usage_vectors(vector):
    from coworker.observability.provider_view import _reply_usage

    entries = [{"body": json.dumps({"usage": vector["raw"]})}]
    assert usage_attrs(_reply_usage(entries)) == {"reported": True, **vector["expected"]}


def test_a_streamed_model_step_is_closed_before_the_engine_sees_the_reply():
    """The engine may add the reply and end the turn before the stream generator runs again;
    the turn's end must then find the model step closed, not close it as interrupted."""
    from coworker.providers.base import StreamChunk

    class Streaming(ScriptedProvider):
        def stream(self, *, model, messages, tools=None, **settings):
            yield StreamChunk(text_delta="hi")
            yield StreamChunk(turn=AssistantTurn(text="hi", finish_reason="stop", usage=TokenUsage(input=3, output=1)))

    rec = Recorder()
    sink = TurnRecording(rec, lambda: "sess_s")
    sink.begin_turn(kind="run", trigger="user", attrs={}, stamp={}, first_message_id=None)
    view = RecordingProvider(Streaming([])).tagged(purpose="helper", sink=sink)
    for chunk in view.stream(model="openai:model-a", messages=[]):
        if chunk.turn is not None:
            [step] = [s for s in rec.as_dict()["steps"] if s["kind"] == "model"]
            assert step["status"] == "ok" and step["attrs"]["finish_reason"] == "stop"
            assert step["attrs"]["ttft_ms"] is not None


def test_the_context_window_is_the_one_the_engine_compacts_against(monkeypatch):
    # Round-2 in-app check: an Ollama model is not in the catalogue, so its steps had no
    # window and the Usage tab hid "Context now". The step records the window the engine
    # uses: the catalogue, else the user's saved size, else Ollama's num_ctx.
    from coworker import model_config
    from coworker.observability.provider_view import _context_window
    from coworker.providers import ollama_context

    monkeypatch.setattr("coworker.providers.matrix.model_context_windows", lambda: {"openai:model-a": 400_000})
    monkeypatch.setattr(model_config, "context_size_for", lambda m: 64_000 if m == "openrouter:model-b" else None)
    monkeypatch.setitem(ollama_context._resolved, "model-c", 40_960)
    assert _context_window("openai:model-a") == 400_000
    assert _context_window("openrouter:model-b") == 64_000
    assert _context_window("ollama:model-c") == 40_960
    assert _context_window("together:model-d") is None


def test_a_saved_window_is_read_after_the_call_not_before(tmp_path, monkeypatch):
    # Reading the user's saved size before the request delayed it (a title call then raced
    # ahead in test_server). The step starts with the catalogue's value only and gets the
    # saved size once the call is over.
    from coworker import model_config

    monkeypatch.setattr(model_config, "context_size_for", lambda model: 64_000)
    engine, rec = _engine(tmp_path, [AssistantTurn(text="hi", finish_reason="stop", usage=None)])
    events = []
    rec.add_listener(lambda ev: events.append(ev))
    _run(engine, "hello")
    [started] = [e for e in events if e.get("type") == "step_started" and e.get("kind") == "model"]
    assert started["attrs"]["context_window"] is None  # not in the catalogue; nothing read yet
    [step] = [s for s in rec.as_dict()["steps"] if s["kind"] == "model"]
    assert step["attrs"]["context_window"] == 64_000