Skip to content

CubeLoop — a Pythonic, async-native agent harness core designed for high performance, readability, and production-grade persistence.

21.6 KB
Raw Blame
"""Phase 2: pin record_content=True attribute emissions + redaction hook.

When ``Tracer(record_content=True)``, the recorder emits opt-in content
attrs on each span layer per the OTel GenAI semconv:

- ``invoke_agent`` (root): gen_ai.input.messages / gen_ai.output.messages
  / gen_ai.system_instructions
- ``cubeloop.turn``: gen_ai.input.messages / gen_ai.output.messages
- ``chat``: gen_ai.system_instructions / gen_ai.input.messages /
  gen_ai.tool.definitions / cubeloop.llm.raw_request / cubeloop.llm.raw_response
- ``execute_tool``: gen_ai.tool.call.arguments / gen_ai.tool.call.result

Plus a ``redact`` hook on the Tracer for per-attribute filtering.
"""

from __future__ import annotations

import json
from typing import Any

from opentelemetry.sdk.trace import ReadableSpan
from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult
from pydantic import BaseModel

from cubeloop.agent.agent import Agent
from cubeloop.agent.types import AgentTool, AgentToolResult
from cubeloop.providers.base import Model, TextContent, ToolCall
from cubeloop.providers.faux import FauxProvider, faux_assistant_message
from cubeloop.tracing import Tracer


MODEL = Model(id="faux-1", provider_id="faux")


class _Capture(SpanExporter):
    def __init__(self) -> None:
        self.spans: list[ReadableSpan] = []

    def export(self, spans):  # noqa: ANN001
        self.spans.extend(spans)
        return SpanExportResult.SUCCESS

    def shutdown(self) -> None:
        pass

    def force_flush(self, timeout_millis: int = 30_000) -> bool:
        return True


def _attrs(span: ReadableSpan) -> dict[str, Any]:
    return dict(span.attributes or {})


def _json_attr(span: ReadableSpan, key: str) -> Any:
    raw = _attrs(span).get(key)
    if raw is None:
        return None
    return json.loads(raw)


async def _build(*, record_content: bool, redact=None, tools=None):
    provider = FauxProvider(provider_id="faux")
    agent = Agent(
        model=provider.model(MODEL.id),
        system_prompt="be helpful, be careful",
        tools=tools,
    )
    exporter = _Capture()
    tracer = Tracer(
        service_name="t",
        agent_name="a",
        exporters=[exporter],
        record_content=record_content,
        redact=redact,
    )
    tracer.attach(agent)
    return agent, provider, exporter, tracer


class TestContentDisabled:
    async def test_no_content_attrs_when_record_content_false(self):
        agent, provider, exporter, tracer = await _build(record_content=False)
        provider.append_responses([faux_assistant_message("hello")])

        await agent.prompt("hi")
        await agent.wait_for_idle()
        await tracer.shutdown()

        for span in exporter.spans:
            attrs = _attrs(span)
            for forbidden in (
                "gen_ai.input.messages",
                "gen_ai.output.messages",
                "gen_ai.system_instructions",
                "gen_ai.tool.definitions",
                "gen_ai.tool.call.arguments",
                "gen_ai.tool.call.result",
                "cubeloop.llm.raw_request",
                "cubeloop.llm.raw_response",
            ):
                assert forbidden not in attrs, (
                    f"{span.name} should not carry {forbidden} with record_content=False"
                )

    async def test_error_details_are_private_when_record_content_false(self):
        secret = "Authorization: Bearer TOPSECRET"
        agent, provider, exporter, tracer = await _build(record_content=False)

        def explode(*_args):
            raise RuntimeError(secret)

        provider.append_responses([explode])

        await agent.prompt("private prompt")
        await agent.wait_for_idle()
        await tracer.shutdown()

        assert any(span.status.status_code.name == "ERROR" for span in exporter.spans)
        for span in exporter.spans:
            assert secret not in (span.status.description or "")
            assert secret not in repr(dict(span.attributes or {}))
            for event in span.events:
                assert secret not in repr(dict(event.attributes or {}))

    async def test_turn_error_message_is_private_when_record_content_false(self):
        secret = "Authorization: Bearer TURNSECRET"
        agent, provider, exporter, tracer = await _build(record_content=False)
        provider.append_responses(
            [
                faux_assistant_message(
                    "oops",
                    stop_reason="error",
                    error_message=secret,
                )
            ]
        )

        await agent.prompt("private prompt")
        await agent.wait_for_idle()
        await tracer.shutdown()

        turn = next(span for span in exporter.spans if span.name == "cubeloop.turn")
        assert turn.status.status_code.name == "ERROR"
        assert turn.status.description == "model error"
        assert secret not in repr(dict(turn.attributes or {}))
        for event in turn.events:
            assert secret not in repr(dict(event.attributes or {}))


class TestRootContent:
    async def test_invoke_agent_records_input_output_system(self):
        agent, provider, exporter, tracer = await _build(record_content=True)
        provider.append_responses([faux_assistant_message("hi back")])

        await agent.prompt("hello there")
        await agent.wait_for_idle()
        await tracer.shutdown()

        root = [s for s in exporter.spans if s.name == "invoke_agent"][0]
        sys_msgs = _json_attr(root, "gen_ai.system_instructions")
        assert sys_msgs == [
            {
                "role": "system",
                "parts": [{"type": "text", "content": "be helpful, be careful"}],
            }
        ]
        inp = _json_attr(root, "gen_ai.input.messages")
        assert isinstance(inp, list)
        assert inp[0]["role"] == "user"
        assert inp[0]["parts"][0] == {"type": "text", "content": "hello there"}
        out = _json_attr(root, "gen_ai.output.messages")
        assert isinstance(out, list)
        assert any(
            m.get("role") == "assistant"
            and any(p.get("content") == "hi back" for p in m.get("parts", []))
            for m in out
        )


class TestChatContent:
    async def test_chat_span_carries_system_input_and_raw_payload(self):
        agent, provider, exporter, tracer = await _build(record_content=True)
        provider.append_responses([faux_assistant_message("hi")])

        await agent.prompt("x")
        await agent.wait_for_idle()
        await tracer.shutdown()

        chat = next(s for s in exporter.spans if s.name.startswith("chat "))
        attrs = _attrs(chat)
        assert "gen_ai.system_instructions" in attrs
        assert "gen_ai.input.messages" in attrs
        raw = json.loads(attrs["cubeloop.llm.raw_request"])
        assert raw["model"] == MODEL.id
        assert "messages" in raw
        resp = json.loads(attrs["cubeloop.llm.raw_response"])
        assert resp["id"] == "faux-1"
        assert resp["role"] == "assistant"

    async def test_provider_exception_details_require_content_opt_in(self):
        secret = "Authorization: Bearer OPTED-IN-ERROR"
        agent, provider, exporter, tracer = await _build(record_content=True)

        def explode(*_args):
            raise RuntimeError(secret)

        provider.append_responses([explode])

        await agent.prompt("private prompt")
        await agent.wait_for_idle()
        await tracer.shutdown()

        chat = next(span for span in exporter.spans if span.name.startswith("chat "))
        exception = next(
            event
            for event in chat.events
            if event.name == "gen_ai.client.operation.exception"
        )
        attrs = dict(exception.attributes or {})
        assert attrs["exception.type"] == "RuntimeError"
        assert attrs["exception.message"] == secret
        assert secret in attrs["exception.stacktrace"]
        assert chat.status.description == secret


class TestToolContent:
    async def test_execute_tool_carries_arguments_and_result(self):
        class Params(BaseModel):
            value: str

        async def run(tool_call_id, params, *, signal=None, on_update=None):
            return AgentToolResult(
                content=[TextContent(text=f"echoed: {params.value}")]
            )

        tool = AgentTool(
            name="echo", description="echo a thing", parameters=Params, execute=run
        )

        agent, provider, exporter, tracer = await _build(
            record_content=True, tools=[tool]
        )
        provider.append_responses(
            [
                faux_assistant_message(
                    [ToolCall(id="t1", name="echo", arguments={"value": "X"})],
                    stop_reason="tool_use",
                ),
                faux_assistant_message("done"),
            ]
        )

        await agent.prompt("x")
        await agent.wait_for_idle()
        await tracer.shutdown()

        t_span = next(s for s in exporter.spans if s.name.startswith("execute_tool "))
        attrs = _attrs(t_span)
        args = json.loads(attrs["gen_ai.tool.call.arguments"])
        assert args == {"value": "X"}
        result = json.loads(attrs["gen_ai.tool.call.result"])
        assert "content" in result
        assert any(
            isinstance(c, dict) and c.get("text") == "echoed: X"
            for c in result.get("content", [])
        )


class TestTurnContent:
    async def test_turn_records_per_turn_input_and_output(self):
        agent, provider, exporter, tracer = await _build(record_content=True)
        provider.append_responses([faux_assistant_message("hello back")])

        await agent.prompt("hi")
        await agent.wait_for_idle()
        await tracer.shutdown()

        turn = next(s for s in exporter.spans if s.name == "cubeloop.turn")
        inp = _json_attr(turn, "gen_ai.input.messages")
        assert inp[0]["parts"][0]["content"] == "hi"
        out = _json_attr(turn, "gen_ai.output.messages")
        assert out[0]["role"] == "assistant"


class TestRootOutputIsOnlyGenerated:
    async def test_root_output_excludes_user_prompts(self):
        """The root invoke_agent's gen_ai.output.messages must be model
        output only (assistant + tool_result), NOT include the caller's
        user prompts — codex round-1 finding on PR #83."""
        agent, provider, exporter, tracer = await _build(record_content=True)
        provider.append_responses([faux_assistant_message("hello back")])
        await agent.prompt("hi there")
        await agent.wait_for_idle()
        await tracer.shutdown()

        root = next(s for s in exporter.spans if s.name == "invoke_agent")
        out = _json_attr(root, "gen_ai.output.messages")
        # No user-role messages must appear in output.
        for m in out:
            assert m["role"] != "user", (
                f"root gen_ai.output.messages must not include user prompts; got {m}"
            )
        # An assistant message with the model output IS expected.
        assert any(
            m["role"] == "assistant"
            and any(p.get("content") == "hello back" for p in m.get("parts", []))
            for m in out
        )


class TestChatInputIncludesPriorAssistant:
    async def test_second_chat_input_includes_prior_assistant_turn(self):
        """In a tool-using multi-turn run, the second chat span's
        gen_ai.input.messages must include the prior assistant
        (tool-call) message + the tool_result so consumers can
        reconstruct the prompt — codex round-1 finding on PR #83."""

        class P(BaseModel):
            pass

        async def run(tool_call_id, params, *, signal=None, on_update=None):
            return AgentToolResult(content=[TextContent(text="tool-output")])

        tool = AgentTool(name="t", description="t", parameters=P, execute=run)

        agent, provider, exporter, tracer = await _build(
            record_content=True, tools=[tool]
        )
        provider.append_responses(
            [
                faux_assistant_message(
                    [ToolCall(id="c1", name="t", arguments={})],
                    stop_reason="tool_use",
                ),
                faux_assistant_message("final answer"),
            ]
        )

        await agent.prompt("kick off")
        await agent.wait_for_idle()
        await tracer.shutdown()

        chats = sorted(
            [s for s in exporter.spans if s.name.startswith("chat ")],
            key=lambda s: s.start_time or 0,
        )
        assert len(chats) == 2
        second = chats[1]
        inp = _json_attr(second, "gen_ai.input.messages")
        roles = [m["role"] for m in inp]
        # Must include the original user prompt, the assistant tool-call
        # message, and the tool result — in that order.
        assert roles.count("user") == 1
        assert "assistant" in roles
        assert "tool" in roles
        # The assistant message must carry the tool_call part.
        asst = next(m for m in inp if m["role"] == "assistant")
        assert any(p.get("type") == "tool_call" for p in asst.get("parts", []))
        # Pin the round-83 codex fix: tool_result must appear exactly
        # once (it was previously appended both by _on_message_start
        # and _on_message_end, producing two identical ``tool`` entries
        # even though the provider context contains only one).
        assert roles.count("tool") == 1


class TestChatInputForContinuedHistory:
    """For ``Agent.resume()`` and other continuation paths, the cubepi
    loop intentionally does NOT emit ``MessageStartEvent`` for the
    pre-existing ``context.messages`` — so without seeding, the
    recorder's transcript starts empty and the first chat span's
    ``gen_ai.input.messages`` omits the conversation history that the
    provider request actually carries.

    Codex P2 finding on PR #83: the recorder must seed the transcript
    from the agent's existing messages at agent_start.
    """

    async def test_resume_chat_input_includes_prior_history(self):
        from cubeloop.providers.base import UserMessage as _U
        from cubeloop.providers.base import AssistantMessage as _A
        from cubeloop.providers.base import ToolResultMessage as _TR

        agent, provider, exporter, tracer = await _build(record_content=True)
        # Pre-load the agent with conversation history via the
        # production state path — ``agent.state.messages`` is what
        # ``_create_context_snapshot`` reads from. The recorder must
        # source its seed from the same place (codex P2 follow-up on
        # PR #87).
        agent.state.messages = [
            _U(content=[TextContent(text="earlier prompt")]),
            _A(
                content=[TextContent(text="ok, working on it")],
                stop_reason="end_turn",
            ),
        ]
        # Now drive a new prompt and check that the FIRST chat span
        # includes the prior turns in its input. (resume() doesn't
        # add a new prompt; using prompt() here is equivalent for the
        # transcript-seeding contract since both code paths leave the
        # pre-existing history un-emitted.)
        provider.append_responses([faux_assistant_message("acknowledged")])
        await agent.prompt("now continue")
        await agent.wait_for_idle()
        await tracer.shutdown()

        chats = sorted(
            [s for s in exporter.spans if s.name.startswith("chat ")],
            key=lambda s: s.start_time or 0,
        )
        assert len(chats) == 1
        inp = _json_attr(chats[0], "gen_ai.input.messages")
        roles = [m["role"] for m in inp]
        # Must carry: earlier user + earlier assistant + new user prompt.
        assert roles == ["user", "assistant", "user"], (
            f"first chat span input must include seeded history; got {roles}"
        )
        # Earlier user content survived.
        assert any("earlier prompt" in p.get("content", "") for p in inp[0]["parts"])
        # Earlier assistant content survived.
        assert any("ok, working on it" in p.get("content", "") for p in inp[1]["parts"])
        # New prompt is last.
        assert any("now continue" in p.get("content", "") for p in inp[2]["parts"])

        # Silence the unused-import linter on the ToolResultMessage
        # alias — it documents the message shapes this test cares about
        # even though only User+Assistant are exercised inline.
        del _TR


class TestRedaction:
    async def test_redact_can_substitute(self):
        seen_keys: list[str] = []

        def redact(key: str, value: Any) -> Any:
            seen_keys.append(key)
            if key == "gen_ai.input.messages":
                return [
                    {
                        "role": "user",
                        "parts": [{"type": "text", "content": "<REDACTED>"}],
                    }
                ]
            return value

        agent, provider, exporter, tracer = await _build(
            record_content=True, redact=redact
        )
        provider.append_responses([faux_assistant_message("ok")])
        await agent.prompt("secret prompt")
        await agent.wait_for_idle()
        await tracer.shutdown()

        assert "gen_ai.input.messages" in seen_keys
        chat = next(s for s in exporter.spans if s.name.startswith("chat "))
        inp = _json_attr(chat, "gen_ai.input.messages")
        assert inp == [
            {"role": "user", "parts": [{"type": "text", "content": "<REDACTED>"}]}
        ]

    async def test_redact_can_drop_attribute(self):
        def redact(key: str, value: Any) -> Any:
            if key == "cubeloop.llm.raw_request":
                return None
            return value

        agent, provider, exporter, tracer = await _build(
            record_content=True, redact=redact
        )
        provider.append_responses([faux_assistant_message("ok")])
        await agent.prompt("x")
        await agent.wait_for_idle()
        await tracer.shutdown()

        chat = next(s for s in exporter.spans if s.name.startswith("chat "))
        assert "cubeloop.llm.raw_request" not in _attrs(chat)
        assert "gen_ai.input.messages" in _attrs(chat)

    async def test_redact_exception_is_swallowed(self):
        def redact(key: str, value: Any) -> Any:
            raise RuntimeError("bad redactor")

        agent, provider, exporter, tracer = await _build(
            record_content=True, redact=redact
        )
        provider.append_responses([faux_assistant_message("ok")])
        await agent.prompt("x")
        await agent.wait_for_idle()
        await tracer.shutdown()

        assert any(s.name == "invoke_agent" for s in exporter.spans)
        chat = next(s for s in exporter.spans if s.name.startswith("chat "))
        assert "gen_ai.input.messages" not in _attrs(chat)


class TestContentHelpers:
    def test_messages_to_semconv_handles_all_block_types(self):
        from cubeloop.providers.base import (
            AssistantMessage,
            ThinkingContent,
        )
        from cubeloop.providers.base import ToolCall as _ToolCall
        from cubeloop.providers.base import (
            ToolResultMessage,
            UserMessage,
        )
        from cubeloop.tracing.content import messages_to_semconv

        msgs = [
            UserMessage(content=[TextContent(text="hi")]),
            AssistantMessage(
                content=[
                    TextContent(text="ok"),
                    ThinkingContent(thinking="planning..."),
                    _ToolCall(id="c1", name="t", arguments={"a": 1}),
                ],
                stop_reason="tool_use",
            ),
            ToolResultMessage(
                tool_call_id="c1",
                tool_name="t",
                content=[TextContent(text="done")],
            ),
        ]
        out = messages_to_semconv(msgs)
        assert out[0]["role"] == "user"
        assert out[1]["role"] == "assistant"
        assert out[1]["parts"][0] == {"type": "text", "content": "ok"}
        assert out[1]["parts"][1] == {
            "type": "reasoning",
            "content": "planning...",
        }
        assert out[1]["parts"][2] == {
            "type": "tool_call",
            "id": "c1",
            "name": "t",
            "arguments": {"a": 1},
        }
        assert out[2]["role"] == "tool"
        assert out[2]["parts"][0] == {
            "type": "tool_call_response",
            "id": "c1",
            "result": "done",
        }

    def test_tool_definitions_anthropic_shape(self):
        from cubeloop.tracing.content import tool_definitions_to_semconv

        payload = {
            "tools": [
                {
                    "name": "fetch",
                    "description": "fetch a url",
                    "input_schema": {"type": "object"},
                }
            ]
        }
        out = tool_definitions_to_semconv(payload)
        assert out == [
            {
                "name": "fetch",
                "description": "fetch a url",
                "parameters": {"type": "object"},
            }
        ]

    def test_tool_definitions_openai_chat_shape(self):
        from cubeloop.tracing.content import tool_definitions_to_semconv

        payload = {
            "tools": [
                {
                    "type": "function",
                    "function": {
                        "name": "fetch",
                        "description": "fetch a url",
                        "parameters": {"type": "object"},
                    },
                }
            ]
        }
        out = tool_definitions_to_semconv(payload)
        assert out == [
            {
                "name": "fetch",
                "description": "fetch a url",
                "parameters": {"type": "object"},
            }
        ]

    def test_serialize_for_attribute_fallback(self):
        from cubeloop.tracing.content import serialize_for_attribute

        class NotJsonable:
            pass

        out = serialize_for_attribute(NotJsonable())
        assert isinstance(out, str)