Building AI agents, atomically
Generic OpenTelemetry tracing: api-only instrumentor, GenAI attributes, LLM attempt spans (#324)
Kenny Vaneetveldeb8506eb910c0
diff --git a/.claude/.codebase-info/.map-state.json b/.claude/.codebase-info/.map-state.jsonindex 0d5046f..461aa09 100644--- a/.claude/.codebase-info/.map-state.json+++ b/.claude/.codebase-info/.map-state.json@@ -2,8 +2,8 @@"schemaVersion": 1,"tool": "codebase-mapper","version": "2.15.10",- "mappedAt": "2026-10-20",- "gitCommit": "609bacb464177649bb73b5d4eea56e631c3c91f0",+ "mappedAt": "2026-10-21",+ "gitCommit": "d1c1bc8a910d3a80a98d08d3a35e49d748611cea","documents": ["architecture.md","coding-style.md",@@ -17,16 +17,16 @@"tech-landscape.md"],"hashes": {- "INDEX.md": "fa005d57802311a643c11e92ea8788e62f8d7eb5eb0001952d26e0e91deb9a49",- "architecture.md": "59d1265c89d3b42671b7c47165408625018537c016a72df421113cae3f80bb70",+ "INDEX.md": "8fec73a52b4a1a7b5c9f2ab7de1e02f273aa49aeb73dc5e29a4d5e7049b2c3ec",+ "architecture.md": "0ff15b04928495b48cdcf40845b5bf9e3bb548c602b734321c4b10f630d83150","coding-style.md": "b63e8dff06d16a2d882f02624c62f490291317aec8b3032f6d79b3ee035e8567",- "communication.md": "4c49f01a99b25a2f56b2b3ab77464f05c7d70d057aa8e998deffe845afc3e7e6",- "dependencies.md": "22b8d208df66c973659a34cea10011a7ac315ddaa65b243c8aaa3adc4eed1a9e",- "directory-structure.md": "4efb931cdefecf2c733f0ffb905ad519aff3a81e245a978c0b62b2c5cf0ad49a",- "entry-points.md": "e03c248c63823276c21dbbe894b98488643ee401c2db136ad2467a00ba57bc9a",- "modules.md": "cd4f81f8ce74dfc4a7bab15ba91dc569245bc342ddf494845d658b51a0a1dc51",+ "communication.md": "0a61b6090f5e67341ad605517111270bdd04caede037e70ec6565de8dcdf5f28",+ "dependencies.md": "abadb58027b4092afaaa0efcaec92893eac7bb598318f7f54a542fd77d607dc1",+ "directory-structure.md": "c8d654506843622fd394e0a998fddba769e4ec534449ccd6755d312192184bfd",+ "entry-points.md": "65ebc321a35f1ae36f587adce8b7b04b71cc4006ec7d518f4c092836c2984705",+ "modules.md": "e1185cdddfbcc7a05a69b80661a6c720aa2a1fc10bc445e319983fffb875524e","onboarding.md": "3d28c671dc095a2aadb297bd7ab200c677779a99c7534e03e55a3d64c935f07e","patterns.md": "09d04b9b8d079ac4ce9e1f5ddafabb6f3faf53b5ef48dca1c2d62d0875e149b0",- "tech-landscape.md": "919d254fae6fa5212a18072ee0d26a1165ce2aeef861faace6c34b8804c025b1"+ "tech-landscape.md": "8a4459f7309dd13fa9e2425507fec35d21b8b3f691bc6a39b6626fffb8d1744d"}}diff --git a/.claude/.codebase-info/INDEX.md b/.claude/.codebase-info/INDEX.mdindex d18ba74..70a6c33 100644--- a/.claude/.codebase-info/INDEX.md+++ b/.claude/.codebase-info/INDEX.md@@ -1,12 +1,12 @@# Codebase Map — Atomic Agents-*Last Updated: 2026-10-20*+*Last Updated: 2026-10-21*Atomic Agents is a lightweight, modular Python framework for building agentic AI applications ascomposable, schema-driven building blocks (built on Instructor + Pydantic). This repository is a`uv`-workspace **monorepo**: the core framework, a TUI tool installer, a tool library, and examples.-**Stack:** Python ≥3.12 · Instructor · Pydantic v2 · LiteLLM · MCP · Textual · uv + Hatchling · Pyright · `VideoURL` multimodal content · optional OpenInference tracing (Phoenix)+**Stack:** Python ≥3.12 · Instructor · Pydantic v2 · LiteLLM · MCP · Textual · uv + Hatchling · Pyright · `VideoURL` multimodal content · optional OpenTelemetry tracing (GenAI attributes; Phoenix/OpenInference optional)**Shape:** Monorepo — `atomic-agents/` (core lib) · `atomic-assembler/` (CLI) · `atomic-forge/` (tools) · `atomic-examples/` (examples)**Package:** `atomic-agents` v2.14.0 · core import package is `atomic_agents`diff --git a/.claude/.codebase-info/architecture.md b/.claude/.codebase-info/architecture.mdindex 4db0b97..97a8b42 100644--- a/.claude/.codebase-info/architecture.md+++ b/.claude/.codebase-info/architecture.md@@ -1,6 +1,6 @@# Architecture-*Last Updated: 2026-10-15*+*Last Updated: 2026-10-21*## Summary@@ -50,7 +50,7 @@ Tool install (atomic-assembler TUI):| Context | `atomic-agents/atomic_agents/context/` | `SystemPromptGenerator`, `BaseDynamicContextProvider`, `BaseChatHistory` (pluggable memory contract) + `ChatHistory` with Instructor media and `VideoURL` || Connectors | `atomic-agents/atomic_agents/connectors/mcp/` | Model Context Protocol tools / resources / prompts || Utils | `atomic-agents/atomic_agents/utils/` | Token counting (LiteLLM), tool-message formatting |-| Observability (opt-in) | `atomic-agents/atomic_agents/observability/` | OpenInference AGENT/TOOL spans for Phoenix / OpenTelemetry; `observability` extra + `openinference_instrumentor` entry point |+| Observability (opt-in) | `atomic-agents/atomic_agents/observability/` | OpenTelemetry AGENT / TOOL / LLM-attempt spans with GenAI attributes for any backend (OpenInference attributes optional via the `phoenix` extra); `observability` extra = `opentelemetry-api`; `openinference_instrumentor` + `opentelemetry_instrumentor` entry points || Assembler (CLI) | `atomic-assembler/atomic_assembler/` | Textual TUI to fetch/install forge tools || Forge (tools) | `atomic-forge/tools/` | 13 standalone tools, each `BaseTool`-based |diff --git a/.claude/.codebase-info/communication.md b/.claude/.codebase-info/communication.mdindex 7325f5e..b4fbce3 100644--- a/.claude/.codebase-info/communication.md+++ b/.claude/.codebase-info/communication.md@@ -1,6 +1,6 @@# Communication & Integrations-*Last Updated: 2026-10-19*+*Last Updated: 2026-10-21*The framework exposes no HTTP API of its own; "communication" here means how it talks to LLMproviders and external tools.@@ -37,15 +37,18 @@ providers and external tools.- Built on Instructor's hook system: `register_hook(event, handler)`, `unregister_hook`,`clear_hooks`. Events include `parse:error`, `completion:kwargs`, `completion:response`,`token:counted`. See the `hooks-example` example and `docs/guides/hooks.md`.-- OpenInference tracing (`atomic_agents.observability`, optional `observability` extra): an AGENT span per- `AtomicAgent` run and a TOOL span per `BaseTool` run, auto-activated by `phoenix.otel.register(auto_instrument=True)`- through `openinference_instrumentor`, by the standard `opentelemetry-instrument` launcher through- `opentelemetry_instrumentor`, or by `AtomicAgentsInstrumentor().instrument(tracer_provider=...)`- for an explicitly configured backend. Both discovery paths share the first installation's provider/config.- Missing optional tracing dependencies are DEBUG-skipped when the launcher passes `skip_dep_check=True`;- manual setup retains its import error and install hint. Choose one configuration path or share the- provider explicitly. LLM spans come from one provider-level OpenInference instrumentor and nest under- the agent span. See `docs/guides/observability.md` and the `phoenix-tracing` example.+- OpenTelemetry tracing (`atomic_agents.observability`, `observability` extra = `opentelemetry-api` only): an AGENT+ span per `AtomicAgent` run, a TOOL span per `BaseTool` run and one CLIENT `chat` span per Instructor attempt+ (driven by Instructor hooks registered once per client), all with OpenTelemetry GenAI attributes; OpenInference+ attributes are added only when its package is importable (`phoenix` extra). Activated by+ `phoenix.otel.register(auto_instrument=True)` through `openinference_instrumentor`, by the `opentelemetry-instrument`+ launcher through `opentelemetry_instrumentor`, or by `AtomicAgentsInstrumentor().instrument(tracer_provider=...,+ config=TracingConfig(...))`. A later install with different explicit values replaces the provider/config in place+ with a WARNING; `None` keeps the installed value. Missing `opentelemetry-api` is DEBUG-skipped when the launcher+ passes `skip_dep_check=True`; manual setup raises `ModuleNotFoundError` with the install hint. Application hooks on+ the same client keep running; `AtomicAgent.clear_hooks()` removes only its own. Set `llm_spans=False` (or+ `ATOMIC_AGENTS_TRACE_LLM_SPANS=false`) when a provider instrumentor also emits LLM spans. See+ `docs/guides/observability.md` and the `phoenix-tracing` example.- Forge tools that run their async body through a `ThreadPoolExecutor` submit via`contextvars.copy_context().run` so the caller's OpenTelemetry context (and the tracing re-entrancy guard)survives in the pool thread.diff --git a/.claude/.codebase-info/dependencies.md b/.claude/.codebase-info/dependencies.mdindex ec1ab3c..a2e76b3 100644--- a/.claude/.codebase-info/dependencies.md+++ b/.claude/.codebase-info/dependencies.md@@ -1,6 +1,6 @@# Dependencies-*Last Updated: 2026-10-19*+*Last Updated: 2026-10-21*Declared in `pyproject.toml` (Hatchling build, `uv` workspace; versions locked in `uv.lock`).@@ -12,15 +12,15 @@ Declared in `pyproject.toml` (Hatchling build, `uv` workspace; versions locked i| litellm (≥1.50,<2) | Provider-agnostic token counting & model metadata || rich (≥13.7.1,<14) | Console formatting |-## Optional extra — `observability`-| Package | Purpose |-|---------|---------|-| openinference-instrumentation (≥0.1.38,<0.2) | `OITracer` / `TraceConfig` used by `atomic_agents.observability` |-| openinference-semantic-conventions (≥0.1.20,<0.2) | Span attribute names (`agent.name` needs ≥0.1.20) |-| opentelemetry-api (≥1.30,<2) | Tracer / context API |+## Optional extras — `observability` and `phoenix`+| Extra | Package | Purpose |+|-------|---------|---------|+| `observability` | opentelemetry-api (≥1.30,<2) | Tracer / context API; the only runtime dependency of `atomic_agents.observability` |+| `phoenix` | opentelemetry-api, openinference-instrumentation (≥0.1.38,<0.2), openinference-semantic-conventions (≥0.1.20,<0.2) | Adds the OpenInference attribute family and `using_session` / `using_user` / `using_metadata`; needed for Arize Phoenix views |-Also listed in the dev group (plus `opentelemetry-sdk`) so CI runs the observability tests. `arize-phoenix-otel`-already brings the two OpenInference packages, so the Phoenix auto path works without the extra.+Users bring their own SDK and exporter (`opentelemetry-sdk`, `opentelemetry-exporter-otlp-proto-http`, ...). The dev+group carries the OpenInference packages plus `opentelemetry-sdk` so the tests run; `arize-phoenix-otel` already+brings the OpenInference packages, so the Phoenix auto path works without the `phoenix` extra.## Runtime — CLI / tooling| Package | Purpose |diff --git a/.claude/.codebase-info/directory-structure.md b/.claude/.codebase-info/directory-structure.mdindex b74bb24..1602c3c 100644--- a/.claude/.codebase-info/directory-structure.md+++ b/.claude/.codebase-info/directory-structure.md@@ -1,6 +1,6 @@# Directory Structure-*Last Updated: 2026-10-15*+*Last Updated: 2026-10-21*## Root Layout@@ -13,7 +13,7 @@ atomic-agents/ # repo root (uv workspace)│ ├── context/ # SystemPromptGenerator, BaseChatHistory/ChatHistory, context providers│ ├── connectors/mcp/ # Model Context Protocol integration│ ├── utils/ # token counter, tool-message formatting-│ └── observability/ # opt-in OpenInference instrumentor (Phoenix / OpenTelemetry)+│ └── observability/ # opt-in OpenTelemetry instrumentor (GenAI attributes; OpenInference optional)│ └── tests/ # pytest suite (agents/, base/, context/, connectors/, observability/, utils/)├── atomic-assembler/ # Textual TUI (`atomic` command) to install forge tools│ └── atomic_assembler/ # main.py, app.py, screens/, widgets/, utils.py, constants.pydiff --git a/.claude/.codebase-info/entry-points.md b/.claude/.codebase-info/entry-points.mdindex a1037d4..bb5feb3 100644--- a/.claude/.codebase-info/entry-points.md+++ b/.claude/.codebase-info/entry-points.md@@ -1,6 +1,6 @@# Entry Points-*Last Updated: 2026-10-19*+*Last Updated: 2026-10-21*## 1. Library API (the primary entry point)@@ -49,7 +49,7 @@ MCP server tools/resources/prompts become agent components via `atomic_agents.co| `fastapi-memory` | multi-user / multi-session memory behind a FastAPI service || `persistent-memory` | custom `BaseChatHistory` backend: cross-session recall via stdlib `sqlite3` (no extra deps) || `hooks-example` | monitoring, error handling, retry via hooks |-| `phoenix-tracing` | Arize Phoenix traces: planner agent -> calculator tool -> answer agent under one CHAIN span, via `register(auto_instrument=True)` |+| `phoenix-tracing` | Arize Phoenix traces: planner agent -> calculator tool -> answer agent under one CHAIN span, via `register(auto_instrument=True)`; depends on `atomic-agents[phoenix]` and sets `ATOMIC_AGENTS_TRACE_LLM_SPANS=false` because the OpenAI instrumentor owns LLM spans there || `basic-multimodal` / `nested-multimodal` | images / PDFs in (nested) schemas || `basic-pdf-analysis` | PDF analysis with a multimodal model || `youtube-summarizer` / `youtube-to-recipe` | transcript → summary / structured recipe |@@ -60,9 +60,10 @@ MCP server tools/resources/prompts become agent components via `atomic_agents.co`atomic_agents.observability:AtomicAgentsInstrumentor` is registered as `atomic_agents` in both`openinference_instrumentor` (Phoenix) and `opentelemetry_instrumentor` (the standard`opentelemetry-instrument` launcher). Both use the same installation state. The launcher needs-its own SDK/exporter setup and the optional OpenInference dependencies; it does not install them-through `opentelemetry-bootstrap`. Set `OTEL_PYTHON_DISABLED_INSTRUMENTATIONS=atomic_agents` to-disable launcher discovery. See `docs/guides/observability.md`.+its own SDK/exporter setup; only `opentelemetry-api` is required for the instrumentor, and+`opentelemetry-bootstrap` does not install Atomic Agents. Set `OTEL_PYTHON_DISABLED_INSTRUMENTATIONS=atomic_agents`+to disable launcher discovery; `ATOMIC_AGENTS_TRACE_*` environment variables configure hiding, size cap and+LLM attempt spans on both automatic paths. See `docs/guides/observability.md`.## Representative Flowdiff --git a/.claude/.codebase-info/modules.md b/.claude/.codebase-info/modules.mdindex ab7154f..f70bbe5 100644--- a/.claude/.codebase-info/modules.md+++ b/.claude/.codebase-info/modules.md@@ -1,6 +1,6 @@# Key Modules-*Last Updated: 2026-10-15*+*Last Updated: 2026-10-21*## Core framework — `atomic-agents/atomic_agents/`@@ -46,19 +46,37 @@### observability (opt-in)- **Location:** `atomic-agents/atomic_agents/observability/`-- **Purpose:** OpenInference tracing for Arize Phoenix and any OpenTelemetry backend. `AtomicAgentsInstrumentor`- (`__init__.py`, stdlib-only imports) lazily loads `openinference_tracing.py`, which patches the four- `AtomicAgent.run*` methods into AGENT spans and every `BaseTool` subclass's `run`/`run_async`/`arun` into TOOL- spans (existing classes via a `__subclasses__` walk, future ones via a wrapped `BaseTool.__init_subclass__`).-- **Contract:** `instrument(tracer_provider=, config=TraceConfig)` / `uninstrument()` /- `is_instrumented_by_opentelemetry`; idempotent and fully reversible. Needs the `observability` extra- (`openinference-instrumentation>=0.1.38`); older versions are refused with a warning so Phoenix's- `register(auto_instrument=True)` (which loads the `openinference_instrumentor` entry point) never fails.-- **Span facts:** `agent.name` = `Class[InputSchema, OutputSchema]`; namespaced `atomic_agents.*` attributes- (model, mode, schema names, history size, context provider names, SHA-256 of the static prompt sections);- streams record the final/last partial and `atomic_agents.stream.closed_early`; a `ContextVar` re-entrancy guard- keeps `run -> arun` / `to_thread` delegation to one TOOL span. Nothing in `atomic_agents/__init__.py` imports it.-- **Tests:** `atomic-agents/tests/observability/` (in-memory span exporter).+- **Purpose:** plain OpenTelemetry tracing for any backend (OTLP collector, Grafana Tempo, Sentry, Datadog,+ Honeycomb, PostHog, Arize Phoenix, Langfuse). Only `opentelemetry-api` is required; the OpenTelemetry GenAI+ semantic-convention attributes are primary and the OpenInference family is an optional layer, auto-on when the+ `openinference-instrumentation` package is importable (`phoenix` extra).+- **Modules:** `__init__.py` (loader, `AtomicAgentsInstrumentor`, stdlib-only imports), `tracing.py` (thin facade),+ `tracing_config.py` (`TracingConfig`, `PayloadVisibility`, env parsing), `span_payloads.py` (serialize, hide before+ serialize, truncate with `<key>.original_length`, bounded/redacted error text), `span_attributes.py` (key constants,+ Instructor-to-semconv provider map, None filtering, best-effort attribute collection, OpenInference context seam,+ `using_conversation`), `agent_spans.py` (four `AtomicAgent.run*` wrappers, stream cursors, run state),+ `tool_spans.py` (`BaseTool` wrappers via `__subclasses__` walk + wrapped `__init_subclass__`, re-entrancy guard),+ `installation.py` (install / uninstall / replace-in-place), `llm_attempt_spans.py` (Instructor hook handlers).+- **Contract:** `instrument(tracer_provider=, config=TracingConfig | PayloadVisibility | None, **options)` /+ `uninstrument()` / `is_instrumented_by_opentelemetry`. Same arguments: no-op; different provider or config:+ replaced in place with a WARNING, `None` keeps the installed value; in-flight spans finish on their original+ provider. Hooks are registered once per Instructor client under a lock and removed on uninstall;+ `suppress_instrumentation` is honored; hooks never raise into user code.+- **Spans:** `invoke_agent {AgentConfig.name or class}` (`gen_ai.operation.name=invoke_agent`, `gen_ai.agent.name`,+ `gen_ai.request.model`, `gen_ai.provider.name`, `gen_ai.conversation.id`, `atomic_agents.agent.class`, `.input`,+ `.output`, model/mode/schema/history/provider/prompt-hash keys, usage totals and `atomic_agents.llm.attempts` at end);+ `execute_tool {tool.tool_name}` (`gen_ai.tool.*`, `atomic_agents.tool.class`); one `chat {model}` CLIENT span per+ Instructor attempt from `completion:kwargs`/`completion:response`/`completion:error` with per-attempt usage+ (cached/reasoning tokens for both chat-completion and Responses shapes), `parse:error` as a `validation_error`+ event; `error.type` on failures; cancellation = `atomic_agents.cancelled`, never ERROR. Spans always attach to the+ current context. `llm_spans=False` keeps totals and drops chat spans.+- **Config:** `TracingConfig(hide_inputs, hide_outputs, max_attribute_length=4096, openinference_attributes=None,+ llm_spans=True)`; env `ATOMIC_AGENTS_TRACE_HIDE_INPUTS/_OUTPUTS` (fallback `OPENINFERENCE_HIDE_*`),+ `ATOMIC_AGENTS_TRACE_MAX_ATTRIBUTE_LENGTH` (tolerant, 4096 fallback), `ATOMIC_AGENTS_TRACE_LLM_SPANS`. Under+ `hide_outputs` every exception becomes `<Type>: <N> attempts (__REDACTED__)`. Nothing in `atomic_agents/__init__.py`+ imports the package.+- **Tests:** `atomic-agents/tests/observability/` (in-memory exporter; `instructor_transport.py` builds real Instructor+ clients over `httpx.MockTransport`; `test_instructor_modes.py` covers TOOLS/JSON/JSON_SCHEMA/RESPONSES_TOOLS).## Subprojectsdiff --git a/.claude/.codebase-info/tech-landscape.md b/.claude/.codebase-info/tech-landscape.mdindex fe1776d..9dcf0ee 100644--- a/.claude/.codebase-info/tech-landscape.md+++ b/.claude/.codebase-info/tech-landscape.md@@ -1,6 +1,6 @@# Technology Landscape-*Last Updated: 2026-10-20*+*Last Updated: 2026-10-21*## Source-of-Truth Files@@ -40,10 +40,12 @@- `code-quality.yml` — on push/PR to `main`/`v2.0`, three jobs: `quality` (`uv sync`,`black --check` and `flake8` across all four project directories, a focused Pyright 1.1.414 constructor check, then`pytest --cov=atomic_agents` for the core); `fresh-install` (build the wheel, install- it with the observability extra into a clean venv without the lockfile, import every module under- `atomic_agents`, which is what a new user's `pip install` sees); `lowest-versions` (resolve runtime- dependencies with `uv pip compile --resolution lowest-direct`, install current pytest tooling plus- `opentelemetry-instrumentation==0.51b0` compatible with the OpenTelemetry API floor, then run the+ it with the `observability` extra plus `opentelemetry-sdk` into a clean venv without the lockfile, prove+ `openinference` is not importable, run a real `instructor.from_openai` client over `httpx.MockTransport`+ through a tool and an agent and assert the GenAI span names, usage keys and the absence of every OpenInference+ key, then import every module under `atomic_agents`); `lowest-versions` (resolve the runtime dependencies plus+ the `observability` and `phoenix` extras with `uv pip compile --resolution lowest-direct`, install current pytest+ tooling plus `opentelemetry-instrumentation==0.51b0` compatible with the OpenTelemetry API floor, then run thesuite). Observability tests exercise real installed discovery metadata, the auto-loader and enabled/disabledlauncher subprocesses with local exporters; no live model/backend calls are needed.- `docs.yml` — on push to `main`: build Sphinx docs, rundiff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.ymlindex 9730e16..ba02e24 100644--- a/.github/workflows/code-quality.yml+++ b/.github/workflows/code-quality.yml@@ -84,6 +84,115 @@ jobs:run: |uv venv fresh-install-venvuv pip install --python fresh-install-venv "atomic-agents[observability] @ $(ls dist/*.whl)"+ uv pip install --python fresh-install-venv opentelemetry-sdk++ - name: Verify OpenInference is absent+ run: |+ if fresh-install-venv/bin/python -c "import openinference"; then+ echo "OpenInference must not be installed in the api-only environment"+ exit 1+ fi++ - name: Trace a tool and agent with only OpenTelemetry+ run: |+ fresh-install-venv/bin/python - <<'PY'+ import httpx+ import instructor+ import openai+ from opentelemetry.sdk.trace import TracerProvider+ from opentelemetry.sdk.trace.export import SimpleSpanProcessor+ from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter++ from atomic_agents import AgentConfig, AtomicAgent, BaseIOSchema, BaseTool+ from atomic_agents.observability import AtomicAgentsInstrumentor, TracingConfig, using_conversation++ exporter = InMemorySpanExporter()+ provider = TracerProvider()+ provider.add_span_processor(SimpleSpanProcessor(exporter))+ AtomicAgentsInstrumentor().instrument(tracer_provider=provider, config=TracingConfig(hide_inputs=True))++ class SmokeInput(BaseIOSchema):+ """Text for the fresh-install tracing check."""++ text: str++ class SmokeOutput(BaseIOSchema):+ """Result of the fresh-install tracing check."""++ text: str++ class SmokeTool(BaseTool[SmokeInput, SmokeOutput]):+ def run(self, params: SmokeInput) -> SmokeOutput:+ return SmokeOutput(text=params.text)++ def respond(request: httpx.Request) -> httpx.Response:+ assert request.url.path == "/v1/chat/completions"+ return httpx.Response(200, json={+ "id": "chatcmpl_smoke", "object": "chat.completion", "created": 1,+ "model": "response-model",+ "choices": [{"index": 0, "finish_reason": "tool_calls", "message": {+ "role": "assistant", "content": None, "tool_calls": [{+ "id": "call_smoke", "type": "function", "function": {+ "name": "SmokeOutput", "arguments": '{"text": "offline response"}'+ }+ }]+ }}],+ "usage": {+ "prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5,+ "prompt_tokens_details": {"cached_tokens": 1},+ "completion_tokens_details": {"reasoning_tokens": 2},+ },+ })++ client = instructor.from_openai(openai.OpenAI(+ api_key="test", http_client=httpx.Client(transport=httpx.MockTransport(respond))+ ))++ class SmokeAgent(AtomicAgent[SmokeInput, SmokeOutput]):+ pass++ assert SmokeTool().run(SmokeInput(text="tool input")).text == "tool input"+ agent = SmokeAgent(config=AgentConfig(client=client, model="offline-model"))+ with using_conversation("smoke-conversation"):+ assert agent.run(SmokeInput(text="private agent input")).text == "offline response"++ spans = exporter.get_finished_spans()+ for span in spans:+ print(f"{span.name}: {sorted(span.attributes)}")+ spans_by_operation = {span.attributes["gen_ai.operation.name"]: span for span in spans}+ assert len(spans) == 3+ assert len(spans_by_operation) == 3+ chat_span = spans_by_operation["chat"]+ tool_span = spans_by_operation["execute_tool"]+ agent_span = spans_by_operation["invoke_agent"]+ assert tool_span.name == f"execute_tool {SmokeTool().tool_name}"+ assert agent_span.name == "invoke_agent SmokeAgent"+ assert tool_span.attributes["gen_ai.tool.name"] == SmokeTool().tool_name+ assert tool_span.attributes["atomic_agents.tool.class"] == "SmokeTool"+ assert agent_span.attributes["atomic_agents.input"] == "__REDACTED__"+ assert agent_span.attributes["gen_ai.agent.name"] == "SmokeAgent"+ assert agent_span.attributes["atomic_agents.llm.attempts"] == 1+ assert chat_span.name == "chat offline-model"+ assert chat_span.parent.span_id == agent_span.context.span_id+ for span in (agent_span, chat_span):+ assert span.attributes["gen_ai.conversation.id"] == "smoke-conversation"+ assert span.attributes["gen_ai.request.model"] == "offline-model"+ assert span.attributes["gen_ai.response.model"] == "response-model"+ assert span.attributes["gen_ai.provider.name"] == "openai"+ assert span.attributes["gen_ai.usage.input_tokens"] == 3+ assert span.attributes["gen_ai.usage.output_tokens"] == 2+ assert span.attributes["gen_ai.usage.cache_read.input_tokens"] == 1+ assert span.attributes["gen_ai.usage.reasoning.output_tokens"] == 2+ forbidden_keys = {+ "input.value", "output.value", "input.mime_type", "output.mime_type",+ "agent.name", "tool.name", "tool.description", "tool.parameters", "session.id",+ }+ for span in spans:+ assert forbidden_keys.isdisjoint(span.attributes)+ assert not any(key.startswith("openinference.") for key in span.attributes)+ client.client.close()+ print("api-only tracing ok: tool, agent and chat spans, real Instructor usage, redacted input, no OpenInference keys")+ PY- name: Import every subpackage the way a new user wouldrun: |@@ -126,7 +235,7 @@ jobs:- name: Resolve runtime dependencies at their declared floorsrun: |echo 'opentelemetry-instrumentation==0.51b0' > telemetry-test-requirements.txt- uv pip compile pyproject.toml telemetry-test-requirements.txt --resolution lowest-direct --extra observability -o lowest-requirements.txt+ uv pip compile pyproject.toml telemetry-test-requirements.txt --resolution lowest-direct --extra observability --extra phoenix -o lowest-requirements.txt- name: Install the floors plus current test toolingrun: |diff --git a/README.md b/README.mdindex 1db7d87..408bc5e 100644--- a/README.md+++ b/README.md@@ -287,7 +287,7 @@ For full, runnable examples, please refer to the following files in the `atomic-In addition to the quickstart examples, we have more complex examples demonstrating the power of Atomic Agents:- [Hooks System](/atomic-examples/hooks-example/README.md): Comprehensive demonstration of the AtomicAgent hook system for monitoring, error handling, and performance metrics with intelligent retry mechanisms.-- [Phoenix Tracing](/atomic-examples/phoenix-tracing/README.md): Traces agents and tools in Arize Phoenix through the optional OpenInference instrumentor (`pip install "atomic-agents[observability]"`).+- [Phoenix Tracing](/atomic-examples/phoenix-tracing/README.md): Traces agents and tools in Arize Phoenix with the OpenTelemetry instrumentor (`pip install "atomic-agents[phoenix]"`).- [Basic Multimodal](/atomic-examples/basic-multimodal/README.md): Demonstrates how to analyze images with text, focusing on extracting structured information from nutrition labels using GPT-4 Vision capabilities.- [Deep Research](/atomic-examples/deep-research/README.md): An advanced example showing how to perform deep research tasks.- [Orchestration Agent](/atomic-examples/orchestration-agent/README.md): Shows how to create an Orchestrator Agent that intelligently decides between using different tools (search or calculator) based on user input.@@ -298,6 +298,10 @@ In addition to the quickstart examples, we have more complex examples demonstratFor a complete list of examples, see the [examples directory](/atomic-examples/).+## Observability++Every agent and tool run is an OpenTelemetry span. Works with any OpenTelemetry backend (Grafana, Sentry, Phoenix, Langfuse, ...), including Datadog, Honeycomb, Jaeger or a plain OTLP collector. Install it with `pip install "atomic-agents[observability]"` and see the [Observability guide](docs/guides/observability.md).+## AI-Assisted DevelopmentBuilding with an AI coding assistant? Atomic Agents ships first-class support so your assistant knows the framework's current API and conventions instead of guessing.diff --git a/atomic-agents/atomic_agents/agents/atomic_agent.py b/atomic-agents/atomic_agents/agents/atomic_agent.pyindex bf31405..76d2423 100644--- a/atomic-agents/atomic_agents/agents/atomic_agent.py+++ b/atomic-agents/atomic_agents/agents/atomic_agent.py@@ -68,6 +68,10 @@ class BasicChatOutputSchema(BaseIOSchema):class AgentConfig(BaseModel):client: instructor.Instructor = Field(..., description="Client for interacting with the language model.")+ name: Optional[str] = Field(+ default=None,+ description="Stable name for this agent in logs and traces, for example the role it plays. Defaults to the class name.",+ )model: str = Field(default="gpt-5-mini", description="The model to use for generating responses.")history: Optional[BaseChatHistory] = Field(default=None, description="History component for storing chat history.")system_prompt_generator: Optional[BaseSystemPromptGenerator] = Field(@@ -190,6 +194,7 @@ class AtomicAgent[InputSchema: BaseIOSchema, OutputSchema: BaseIOSchema]:Args:config (AgentConfig): Configuration for the chat agent."""+ self.name = config.nameself.client = config.clientself.model = config.modelself.history = config.history if config.history is not None else ChatHistory()@@ -794,16 +799,11 @@ class AtomicAgent[InputSchema: BaseIOSchema, OutputSchema: BaseIOSchema]:Args:event (Optional[str]): The event name to clear, or None to clear all"""- if event:- if event in self._hook_handlers:- # Clear from instructor client first- if hasattr(self.client, "clear"):- self.client.clear(event)- self._hook_handlers[event].clear()- else:- # Clear all hooks- if hasattr(self.client, "clear"):- self.client.clear()+ events = [event] if event else list(self._hook_handlers)+ for registered_event in events:+ for handler in list(self._hook_handlers.get(registered_event, [])):+ self.unregister_hook(registered_event, handler)+ if not event:self._hook_handlers.clear()def _dispatch_hook(self, event: str, *args, **kwargs) -> None:diff --git a/atomic-agents/atomic_agents/observability/__init__.py b/atomic-agents/atomic_agents/observability/__init__.pyindex 19bb0e2..254230a 100644--- a/atomic-agents/atomic_agents/observability/__init__.py+++ b/atomic-agents/atomic_agents/observability/__init__.py@@ -1,34 +1,34 @@-"""-Opt-in OpenInference tracing for Atomic Agents.+"""Opt-in tracing for any OpenTelemetry backend using GenAI attributes.-Emits an AGENT span per ``AtomicAgent`` run and a TOOL span per ``BaseTool`` run, rendered by-Arize Phoenix and any other OpenTelemetry backend. Needs the ``observability`` extra:-``pip install "atomic-agents[observability]"``. Phoenix discovers :class:`AtomicAgentsInstrumentor`-through the ``openinference_instrumentor`` entry point, so ``phoenix.otel.register(auto_instrument=True)``-activates it without further code. See ``docs/guides/observability.md``.+Install the observability extra for tracing. The optional phoenix extra adds+Arize Phoenix integration and OpenInference attributes."""import importlib.utilimport loggingfrom importlib.metadata import PackageNotFoundError, version-from typing import TYPE_CHECKING, Optional+from typing import TYPE_CHECKINGif TYPE_CHECKING:- from openinference.instrumentation import TraceConfig+ from atomic_agents.observability.tracing import PayloadVisibility, TracingConfig, using_conversationfrom opentelemetry.trace import TracerProviderlogger = logging.getLogger(__name__)_INSTALL_HINT = (- "Atomic Agents observability needs the OpenInference packages. "- 'Install them with: pip install "atomic-agents[observability]"'+ "Atomic Agents tracing needs opentelemetry-api. "+ 'Install it with: pip install "atomic-agents[observability]" '+ '(add "atomic-agents[phoenix]" for Arize Phoenix / OpenInference attributes)')_OPENINFERENCE_DISTRIBUTION = "openinference-instrumentation"_MINIMUM_OPENINFERENCE_VERSION = (0, 1, 38)-def _openinference_is_installed() -> bool:- return importlib.util.find_spec("openinference") is not None+def _opentelemetry_is_installed() -> bool:+ try:+ return importlib.util.find_spec("opentelemetry.trace") is not None+ except ModuleNotFoundError:+ return Falsedef _openinference_is_too_old() -> bool:@@ -41,21 +41,26 @@ def _openinference_is_too_old() -> bool:def _load_tracing_module():- if _openinference_is_too_old():- logger.warning(- "Atomic Agents tracing is disabled: %s %s or newer is required.",- _OPENINFERENCE_DISTRIBUTION,- ".".join(str(part) for part in _MINIMUM_OPENINFERENCE_VERSION),- )- return None- from atomic_agents.observability import openinference_tracing+ from atomic_agents.observability import tracing++ return tracing+++def __getattr__(name: str) -> object:+ if name in ("TracingConfig", "using_conversation"):+ return getattr(_load_tracing_module(), name)+ raise AttributeError(name)+- return openinference_tracing+def _warn_unknown_options(options: dict[str, object]) -> None:+ for key in options:+ if key != "skip_dep_check":+ logger.warning("Unknown Atomic Agents instrumentation option %s ignored", key)class AtomicAgentsInstrumentor:"""- Installs OpenInference AGENT and TOOL spans around ``AtomicAgent`` and ``BaseTool`` calls.+ Installs AGENT, TOOL and LLM-attempt spans around ``AtomicAgent`` and ``BaseTool`` calls.The public surface mirrors OpenTelemetry's ``BaseInstrumentor`` (``instrument`` / ``uninstrument``)without depending on it, because Arize Phoenix only guarantees ``opentelemetry-api`` and@@ -65,31 +70,30 @@ class AtomicAgentsInstrumentor:def instrument(self,*,- tracer_provider: Optional["TracerProvider"] = None,- config: Optional["TraceConfig"] = None,+ tracer_provider: "TracerProvider | None" = None,+ config: "TracingConfig | PayloadVisibility | None" = None,**options: object,) -> None:"""Patch the framework classes. Safe to call more than once; a second call with different- arguments logs a warning and keeps the first installation.+ arguments replaces the tracer provider/config and logs a warning.Args:tracer_provider: An OpenTelemetry ``TracerProvider``. Defaults to the global provider.- config: An ``openinference.instrumentation.TraceConfig`` (for example ``hide_inputs=True``).+ config: Payload visibility and truncation settings. OpenInference TraceConfig is also accepted.**options: Launcher options accepted for OpenTelemetry auto-instrumentation compatibility.Raises:ModuleNotFoundError: When the ``observability`` extra is not installed, except duringOpenTelemetry auto-discovery, which skips unavailable optional instrumentation."""- if not _openinference_is_installed():+ _warn_unknown_options(options)+ if not _opentelemetry_is_installed():if options.get("skip_dep_check") is True:logger.debug(_INSTALL_HINT)returnraise ModuleNotFoundError(_INSTALL_HINT)- tracing = _load_tracing_module()- if tracing is not None:- tracing.install(tracer_provider=tracer_provider, config=config)+ _load_tracing_module().install(tracer_provider=tracer_provider, config=config)def uninstrument(self) -> None:"""Restore the original framework methods. No-op when nothing is installed."""@@ -99,9 +103,9 @@ class AtomicAgentsInstrumentor:@propertydef is_instrumented_by_opentelemetry(self) -> bool:- if not _openinference_is_installed() or _openinference_is_too_old():+ if not _opentelemetry_is_installed():return Falsereturn _load_tracing_module().is_installed()-__all__ = ["AtomicAgentsInstrumentor"]+__all__ = ["AtomicAgentsInstrumentor", "TracingConfig", "using_conversation"]diff --git a/atomic-agents/atomic_agents/observability/agent_spans.py b/atomic-agents/atomic_agents/observability/agent_spans.pynew file mode 100644index 0000000..d49000f--- /dev/null+++ b/atomic-agents/atomic_agents/observability/agent_spans.py@@ -0,0 +1,239 @@+"""Agent call and stream spans."""++from __future__ import annotations++import asyncio+import functools+from contextlib import contextmanager+from typing import AsyncGenerator, Awaitable, Callable, Coroutine, Generator, Generic, Iterator, TypeVar++from opentelemetry import context as context_api+from opentelemetry import trace as trace_api+from opentelemetry.trace import Span, Status, StatusCode++from atomic_agents.agents.atomic_agent import AtomicAgent++from . import installation, llm_attempt_spans+from .span_attributes import ATTR_STREAM_CLOSED_EARLY, _agent_display_name, _agent_start_attributes, _suppressed+from .span_payloads import _record_failure, _set_input_safely, _set_output_safely++OutputSchema = TypeVar("OutputSchema")+CallResult = TypeVar("CallResult")++AGENT_SYNC_METHOD = "run"+AGENT_ASYNC_METHOD = "run_async"+AGENT_SYNC_STREAM_METHOD = "run_stream"+AGENT_ASYNC_STREAM_METHOD = "run_async_stream"+++def _user_input_from_call(args: tuple[object, ...], kwargs: dict[str, object]) -> object:+ return args[0] if args else kwargs.get("user_input")+++def _finish_agent_span(span: Span, output: object) -> None:+ _set_output_safely(span, output)+ span.set_status(Status(StatusCode.OK))+++def _mark_closed_early(span: Span, output: object) -> None:+ span.set_attribute(ATTR_STREAM_CLOSED_EARLY, True)+ _set_output_safely(span, output)+++@contextmanager+def _agent_span(agent: AtomicAgent, args: tuple[object, ...], kwargs: dict[str, object]) -> Iterator[Span]:+ installed = installation._current_installation()+ with installed.tracer.start_as_current_span(+ f"invoke_agent {_agent_display_name(agent)}",+ attributes=_agent_start_attributes(agent),+ record_exception=False,+ set_status_on_exception=False,+ ) as span:+ _set_input_safely(span, _user_input_from_call(args, kwargs))+ run = llm_attempt_spans.new_run(span, agent)+ try:+ with llm_attempt_spans.active_run(run):+ yield span+ except (asyncio.CancelledError, GeneratorExit):+ span.set_attribute("atomic_agents.cancelled", True)+ raise+ except BaseException as error:+ _record_failure(span, error)+ raise+ finally:+ llm_attempt_spans.finish_run(run)+++def _trace_agent_call(wrapped: Callable[..., OutputSchema]) -> Callable[..., OutputSchema]:+ @functools.wraps(wrapped)+ def traced(agent: AtomicAgent, *args: object, **kwargs: object) -> OutputSchema:+ if _suppressed():+ return wrapped(agent, *args, **kwargs)+ with _agent_span(agent, args, kwargs) as span:+ response = wrapped(agent, *args, **kwargs)+ _finish_agent_span(span, response)+ return response++ return traced+++def _trace_agent_async_call(+ wrapped: Callable[..., Awaitable[OutputSchema]],+) -> Callable[..., Coroutine[object, object, OutputSchema]]:+ @functools.wraps(wrapped)+ async def traced(agent: AtomicAgent, *args: object, **kwargs: object) -> OutputSchema:+ if _suppressed():+ return await wrapped(agent, *args, **kwargs)+ with _agent_span(agent, args, kwargs) as span:+ response = await wrapped(agent, *args, **kwargs)+ _finish_agent_span(span, response)+ return response++ return traced+++def _start_agent_span(agent: AtomicAgent, args: tuple[object, ...], kwargs: dict[str, object]) -> Span:+ if _suppressed():+ return trace_api.INVALID_SPAN+ span = installation._current_installation().tracer.start_span(+ f"invoke_agent {_agent_display_name(agent)}",+ attributes=_agent_start_attributes(agent),+ )+ _set_input_safely(span, _user_input_from_call(args, kwargs))+ return span+++def _call_within_span(run: llm_attempt_spans._AgentRun, function: Callable[..., CallResult], *args: object) -> CallResult:+ span = run.span+ token = context_api.attach(trace_api.set_span_in_context(span))+ try:+ with llm_attempt_spans.active_run(run):+ return function(*args)+ finally:+ context_api.detach(token)+++async def _await_within_span(run: llm_attempt_spans._AgentRun, awaitable: Awaitable[CallResult]) -> CallResult:+ span = run.span+ token = context_api.attach(trace_api.set_span_in_context(span))+ try:+ with llm_attempt_spans.active_run(run):+ return await awaitable+ finally:+ context_api.detach(token)+++class _SyncStreamCursor(Generic[OutputSchema]):+ def __init__(self, run: llm_attempt_spans._AgentRun, inner: Generator[OutputSchema, None, OutputSchema | None]):+ self.run = run+ self.span = run.span+ self.inner = inner+ self.last: OutputSchema+ self.output: OutputSchema | None = None+ self.final: OutputSchema | None = None++ def advance(self) -> bool:+ try:+ self.last = _call_within_span(self.run, next, self.inner)+ self.output = self.last+ except StopIteration as stop:+ self.final = stop.value+ return False+ return True++ def close_early(self) -> None:+ _call_within_span(self.run, self.inner.close)+ _mark_closed_early(self.span, self.output)+ self.span.set_status(Status(StatusCode.OK))+++class _AsyncStreamCursor(Generic[OutputSchema]):+ """Async twin of :class:`_SyncStreamCursor`; async generators carry no return value, so the last partial is the output."""++ def __init__(self, run: llm_attempt_spans._AgentRun, inner: AsyncGenerator[OutputSchema, None]):+ self.run = run+ self.span = run.span+ self.inner = inner+ self.last: OutputSchema+ self.output: OutputSchema | None = None++ async def advance(self) -> bool:+ try:+ self.last = await _await_within_span(self.run, self.inner.__anext__())+ self.output = self.last+ except StopAsyncIteration:+ return False+ return True++ async def close_early(self) -> None:+ await _await_within_span(self.run, self.inner.aclose())+ _mark_closed_early(self.span, self.output)+ self.span.set_status(Status(StatusCode.OK))++ def mark_cancelled(self) -> None:+ _mark_closed_early(self.span, self.output)+++def _trace_agent_stream(+ wrapped: Callable[..., Generator[OutputSchema, None, OutputSchema | None]],+) -> Callable[..., Generator[OutputSchema, None, OutputSchema | None]]:+ @functools.wraps(wrapped)+ def traced(agent: AtomicAgent, *args: object, **kwargs: object) -> Generator[OutputSchema, None, OutputSchema | None]:+ span = _start_agent_span(agent, args, kwargs)+ run = llm_attempt_spans.new_run(span, agent)+ cursor = _SyncStreamCursor(run, wrapped(agent, *args, **kwargs))+ try:+ while cursor.advance():+ yield cursor.last+ _finish_agent_span(span, cursor.final)+ return cursor.final+ except GeneratorExit:+ cursor.close_early()+ raise+ except asyncio.CancelledError:+ _mark_closed_early(span, cursor.output)+ raise+ except BaseException as error:+ _record_failure(span, error)+ raise+ finally:+ llm_attempt_spans.finish_run(run)+ span.end()++ return traced+++def _trace_agent_async_stream(+ wrapped: Callable[..., AsyncGenerator[OutputSchema, None]],+) -> Callable[..., AsyncGenerator[OutputSchema, None]]:+ @functools.wraps(wrapped)+ async def traced(agent: AtomicAgent, *args: object, **kwargs: object) -> AsyncGenerator[OutputSchema, None]:+ span = _start_agent_span(agent, args, kwargs)+ run = llm_attempt_spans.new_run(span, agent)+ cursor = _AsyncStreamCursor(run, wrapped(agent, *args, **kwargs))+ try:+ while await cursor.advance():+ yield cursor.last+ _finish_agent_span(span, cursor.output)+ except GeneratorExit:+ await cursor.close_early()+ raise+ except asyncio.CancelledError:+ cursor.mark_cancelled()+ raise+ except BaseException as error:+ _record_failure(span, error)+ raise+ finally:+ llm_attempt_spans.finish_run(run)+ span.end()++ return traced+++_AGENT_WRAPPERS = {+ AGENT_SYNC_METHOD: _trace_agent_call,+ AGENT_ASYNC_METHOD: _trace_agent_async_call,+ AGENT_SYNC_STREAM_METHOD: _trace_agent_stream,+ AGENT_ASYNC_STREAM_METHOD: _trace_agent_async_stream,+}diff --git a/atomic-agents/atomic_agents/observability/installation.py b/atomic-agents/atomic_agents/observability/installation.pynew file mode 100644index 0000000..8805235--- /dev/null+++ b/atomic-agents/atomic_agents/observability/installation.py@@ -0,0 +1,130 @@+"""Tracing installation and method restoration."""++import logging+from dataclasses import replace+from typing import TYPE_CHECKING, Callable, Optional, Type++from opentelemetry import trace as trace_api+from opentelemetry.trace import Tracer+from opentelemetry.util.types import AttributeValue++from atomic_agents import __version__+from atomic_agents.agents.atomic_agent import AtomicAgent+from atomic_agents.base.base_tool import BaseTool++from . import agent_spans, llm_attempt_spans, span_attributes, tool_spans+from .tracing_config import PayloadVisibility, TracingConfig++if TYPE_CHECKING:+ from opentelemetry.trace import TracerProvider+++logger = logging.getLogger("atomic_agents.observability.tracing")+++TRACED_MARKER = "__atomic_agents_traced__"+INHERITED_MARKER = "__atomic_agents_inherited__"+TRACER_NAME = "atomic_agents"+++def _current_installation() -> "_Installation":+ assert _installation is not None+ return _installation+++class _Installation:+ def __init__(+ self,+ tracer: Tracer,+ tracer_provider: Optional["TracerProvider"],+ config: TracingConfig,+ original_init_subclass: classmethod,+ context_attributes: Callable[[], dict[str, AttributeValue]] | None,+ ):+ self.context_attributes = context_attributes+ self.tracer = tracer+ self.tracer_provider = tracer_provider+ self.config = config+ self.original_init_subclass = original_init_subclass+++_installation: Optional[_Installation] = None+++def is_installed() -> bool:+ return _installation is not None+++def install(+ *, tracer_provider: Optional["TracerProvider"] = None, config: TracingConfig | PayloadVisibility | None = None+) -> None:+ global _installation+ if _installation is not None:+ tracer_provider = _installation.tracer_provider if tracer_provider is None else tracer_provider+ _replace_installation(_installation, tracer_provider, config)+ return+ config = TracingConfig.coerce(config)+ context_attributes = span_attributes._import_context_attributes()+ if config.openinference_attributes is None:+ config = replace(config, openinference_attributes=context_attributes is not None)+ tracer = trace_api.get_tracer(TRACER_NAME, __version__, tracer_provider)+ original_init_subclass = BaseTool.__dict__["__init_subclass__"]+ _installation = _Installation(tracer, tracer_provider, config, original_init_subclass, context_attributes)+ _instrument_agent_class()+ tool_spans._instrument_existing_tool_classes()+ tool_spans._hook_future_tool_classes(original_init_subclass)+++def uninstall() -> None:+ global _installation+ if _installation is None:+ return+ setattr(BaseTool, "__init_subclass__", _installation.original_init_subclass)+ for tool_class in tool_spans._all_subclasses(BaseTool):+ tool_spans._restore_tool_class(tool_class)+ for method_name in agent_spans._AGENT_WRAPPERS:+ _restore_method(AtomicAgent, method_name)+ llm_attempt_spans.unregister_clients()+ _installation = None+++def _replace_installation(+ installed: _Installation, tracer_provider: Optional["TracerProvider"], config: TracingConfig | PayloadVisibility | None+) -> None:+ config = installed.config if config is None else TracingConfig.coerce(config)+ if config.openinference_attributes is None:+ config = replace(config, openinference_attributes=installed.config.openinference_attributes)+ if tracer_provider is installed.tracer_provider and config == installed.config:+ return+ installed.tracer = trace_api.get_tracer(TRACER_NAME, __version__, tracer_provider)+ installed.tracer_provider = tracer_provider+ installed.config = config+ installed.context_attributes = span_attributes._import_context_attributes()+ logger.warning(+ "AtomicAgentsInstrumentor tracer provider/config replaced; "+ "in-flight spans finish on the provider they started with."+ )+++def _instrument_agent_class() -> None:+ for method_name, wrap in agent_spans._AGENT_WRAPPERS.items():+ original = AtomicAgent.__dict__[method_name]+ if not getattr(original, TRACED_MARKER, False):+ setattr(AtomicAgent, method_name, _mark_traced(wrap(original), original, False))+++def _mark_traced(wrapper: Callable, original: Callable, inherited: bool) -> Callable:+ setattr(wrapper, TRACED_MARKER, True)+ setattr(wrapper, INHERITED_MARKER, inherited)+ setattr(wrapper, "__wrapped__", original)+ return wrapper+++def _restore_method(owner: Type, method_name: str) -> None:+ current = owner.__dict__.get(method_name)+ if current is None or not getattr(current, TRACED_MARKER, False):+ return+ if getattr(current, INHERITED_MARKER, False):+ delattr(owner, method_name)+ else:+ setattr(owner, method_name, current.__wrapped__)diff --git a/atomic-agents/atomic_agents/observability/llm_attempt_spans.py b/atomic-agents/atomic_agents/observability/llm_attempt_spans.pynew file mode 100644index 0000000..05e4deb--- /dev/null+++ b/atomic-agents/atomic_agents/observability/llm_attempt_spans.py@@ -0,0 +1,295 @@+"""Instructor attempt spans and cumulative agent usage."""++from __future__ import annotations++import contextvars+import functools+import logging+import threading+import weakref+from contextlib import contextmanager+from dataclasses import dataclass, field+from typing import Callable, Iterator, ParamSpec, Protocol++from opentelemetry import context as context_api, trace as trace_api+from opentelemetry.trace import Span, SpanKind, Status, StatusCode, Tracer+from opentelemetry.util.types import AttributeValue++from atomic_agents.agents.atomic_agent import AtomicAgent++from . import installation+from .span_attributes import _best_effort_attributes, _optional_attributes, _provider_attributes, _without_none+from .span_payloads import _bounded_text, _failure_message, _record_failure, _write_payload+from .tracing_config import TracingConfig++logger = logging.getLogger("atomic_agents.observability.tracing")+_PARAMETERS = ParamSpec("_PARAMETERS")+_RESPONSE_MODEL = "gen_ai.response.model"+_ATTEMPT = "atomic_agents.llm.attempt"+_VALIDATION_ERROR = "atomic_agents.llm.last_validation_error"+++class _PromptTokenDetails(Protocol):+ cached_tokens: int | None+++class _CompletionTokenDetails(Protocol):+ reasoning_tokens: int | None+++class _InputOutputUsage(Protocol):+ input_tokens: int+ output_tokens: int+ input_tokens_details: _PromptTokenDetails | None+ output_tokens_details: _CompletionTokenDetails | None+++class _PromptCompletionUsage(Protocol):+ prompt_tokens: int+ completion_tokens: int+ prompt_tokens_details: _PromptTokenDetails | None+ completion_tokens_details: _CompletionTokenDetails | None+++_Usage = _InputOutputUsage | _PromptCompletionUsage+++class _Response(Protocol):+ model: str+ usage: _Usage+++class _HookClient(Protocol):+ def on(self, event: str, handler: Callable[..., None]) -> object:+ """Append a handler without replacing application hooks."""++ def off(self, event: str, handler: Callable[..., None]) -> object:+ """Remove only this handler, preserving application hooks."""+++@dataclass+class _AgentRun:+ span: Span+ tracer: Tracer+ config: TracingConfig+ provider: dict[str, AttributeValue]+ usage_totals: dict[str, int] = field(default_factory=dict)+ attempt_count: int = 0+ suppressed_attempt: bool = False+ response_received: bool = False+ open_attempt: Span | None = None+ context_token: contextvars.Token | None = None+++_active_run: contextvars.ContextVar[_AgentRun | None] = contextvars.ContextVar("atomic_agents_llm_run", default=None)+_clients: weakref.WeakSet[_HookClient] = weakref.WeakSet()+_registration_lock = threading.Lock()+++def _never_raise(handler: Callable[_PARAMETERS, None]) -> Callable[_PARAMETERS, None]:+ @functools.wraps(handler)+ def guarded(*args: _PARAMETERS.args, **kwargs: _PARAMETERS.kwargs) -> None:+ try:+ handler(*args, **kwargs)+ except Exception:+ logger.warning("Instructor tracing hook failed: %s", handler.__name__, exc_info=True)++ return guarded+++@_never_raise+def register_client(client: _HookClient) -> None:+ if not hasattr(client, "on"):+ return+ # Shared clients must acquire all handlers once before another run can use them.+ with _registration_lock:+ if client not in _clients:+ _register_handlers(client)+ _clients.add(client)+++def _register_handlers(client: _HookClient) -> None:+ registered: list[tuple[str, Callable[..., None]]] = []+ try:+ for event, handler in _HANDLERS.items():+ client.on(event, handler)+ registered.append((event, handler))+ except Exception:+ for event, handler in registered:+ client.off(event, handler)+ raise+++@_never_raise+def unregister_clients() -> None:+ with _registration_lock:+ for client in list(_clients):+ _unregister_client(client)+ _clients.clear()+++@_never_raise+def _unregister_client(client: _HookClient) -> None:+ for event, handler in _HANDLERS.items():+ client.off(event, handler)+++def new_run(span: Span, agent: AtomicAgent) -> _AgentRun:+ installed = installation._current_installation()+ register_client(getattr(agent, "client"))+ provider = _best_effort_attributes(+ {"gen_ai.provider.name": lambda: _provider_attributes(agent).get("gen_ai.provider.name")}+ )+ return _AgentRun(span, installed.tracer, installed.config, provider)+++@contextmanager+def active_run(run: _AgentRun) -> Iterator[None]:+ token = _active_run.set(None if run.span is trace_api.INVALID_SPAN else run)+ _attach_attempt(run)+ try:+ yield+ finally:+ _detach_attempt(run)+ _active_run.reset(token)+++def _attach_attempt(run: _AgentRun) -> None:+ if run.open_attempt is not None:+ run.context_token = context_api.attach(trace_api.set_span_in_context(run.open_attempt))+++def _detach_attempt(run: _AgentRun) -> None:+ if run.context_token is not None:+ context_api.detach(run.context_token)+ run.context_token = None+++@_never_raise+def finish_attempt(run: _AgentRun, status: Status | None = None) -> None:+ if run.open_attempt is None:+ return+ if run.response_received:+ status = Status(StatusCode.OK)+ run.response_received = False+ if status is not None:+ run.open_attempt.set_status(status)+ run.open_attempt.end()+ run.open_attempt = None+ _detach_attempt(run)+++@_never_raise+def _completion_kwargs(*args: object, **kwargs: object) -> None:+ run = _active_run.get()+ if run is None:+ return+ finish_attempt(run)+ # Suppressed kwargs must also swallow later response events outside that context.+ run.suppressed_attempt = bool(context_api.get_value("suppress_instrumentation"))+ if run.suppressed_attempt:+ return+ run.attempt_count += 1+ if not run.config.llm_spans:+ return+ model = _request_model(kwargs)+ attributes: dict[str, AttributeValue | None] = {+ "gen_ai.operation.name": "chat",+ "gen_ai.request.model": model,+ _ATTEMPT: run.attempt_count,+ }+ attributes.update(run.provider)+ attributes.update(_optional_attributes("LLM", {}))+ run.open_attempt = run.tracer.start_span(+ f"chat {model}" if model else "chat", kind=SpanKind.CLIENT, attributes=_without_none(attributes)+ )+ _attach_attempt(run)+++def _request_model(kwargs: dict[str, object]) -> str | None:+ model = kwargs.get("model")+ return str(model) if model is not None else None+++def _usage_attributes(usage: _Usage | None) -> dict[str, int]:+ readers = {+ "input_tokens": getattr(usage, "input_tokens", getattr(usage, "prompt_tokens", None)),+ "output_tokens": getattr(usage, "output_tokens", getattr(usage, "completion_tokens", None)),+ "cache_read.input_tokens": getattr(+ getattr(usage, "input_tokens_details", getattr(usage, "prompt_tokens_details", None)), "cached_tokens", None+ ),+ "reasoning.output_tokens": getattr(+ getattr(usage, "output_tokens_details", getattr(usage, "completion_tokens_details", None)),+ "reasoning_tokens",+ None,+ ),+ }+ return {"gen_ai.usage." + key: value for key, value in readers.items() if isinstance(value, int)}+++@_never_raise+def _completion_response(response: _Response, **kwargs: object) -> None:+ run = _active_run.get()+ if run is None or run.suppressed_attempt:+ return+ attributes = _without_none({_RESPONSE_MODEL: getattr(response, "model", None)})+ run.span.set_attributes(attributes)+ usage = _usage_attributes(getattr(response, "usage", None))+ for key, value in usage.items():+ run.usage_totals[key] = run.usage_totals.get(key, 0) + value+ _complete_response_attempt(run, usage, attributes)+++def _complete_response_attempt(run: _AgentRun, usage: dict[str, int], attributes: dict[str, AttributeValue]) -> None:+ if run.open_attempt is None:+ return+ run.open_attempt.set_attributes(attributes)+ run.open_attempt.set_attributes(usage)+ run.response_received = True+++@_never_raise+def _completion_error(error: BaseException, **kwargs: object) -> None:+ run = _active_run.get()+ if run is None or run.suppressed_attempt or run.open_attempt is None:+ return+ run.response_received = False+ _record_failure(run.open_attempt, error, run.config)+ finish_attempt(run)+++@_never_raise+def _parse_error(error: BaseException, **kwargs: object) -> None:+ run = _active_run.get()+ if run is None or run.suppressed_attempt:+ return+ text = "__REDACTED__" if run.config.hide_outputs else _failure_message(error, run.config)+ _write_payload(run.span, _VALIDATION_ERROR, None, text, run.config.hide_outputs, run.config)+ event = {_ATTEMPT: run.attempt_count, _VALIDATION_ERROR: _bounded_text(text, run.config)}+ run.span.add_event("validation_error", event)+ _mark_attempt_validation_error(run, error)+++def _mark_attempt_validation_error(run: _AgentRun, error: BaseException) -> None:+ if run.open_attempt is None:+ return+ run.response_received = False+ message = _bounded_text(_failure_message(error, run.config), run.config)+ run.open_attempt.add_event("validation_error", {_ATTEMPT: run.attempt_count, _VALIDATION_ERROR: message})+ run.open_attempt.set_attribute("error.type", type(error).__qualname__)+ run.open_attempt.set_status(Status(StatusCode.ERROR, message))+++@_never_raise+def finish_run(run: _AgentRun) -> None:+ finish_attempt(run)+ run.span.set_attributes(run.usage_totals)+ run.span.set_attribute("atomic_agents.llm.attempts", run.attempt_count)+++_HANDLERS: dict[str, Callable[..., None]] = {+ "completion:kwargs": _completion_kwargs,+ "completion:response": _completion_response,+ "completion:error": _completion_error,+ "parse:error": _parse_error,+}diff --git a/atomic-agents/atomic_agents/observability/openinference_tracing.py b/atomic-agents/atomic_agents/observability/openinference_tracing.pydeleted file mode 100644index 7959be4..0000000--- a/atomic-agents/atomic_agents/observability/openinference_tracing.py+++ /dev/null@@ -1,475 +0,0 @@-"""-OpenInference span emission for ``AtomicAgent`` runs and ``BaseTool`` runs.--Loaded lazily by :class:`atomic_agents.observability.AtomicAgentsInstrumentor`; importing this module-requires the ``observability`` extra.-"""--import asyncio-import contextvars-import functools-import hashlib-import inspect-import json-import logging-from types import FunctionType-from typing import TYPE_CHECKING, Any, AsyncGenerator, Callable, Dict, Generator, List, Optional, Type--from openinference.instrumentation import OITracer, TraceConfig-from openinference.semconv.trace import SpanAttributes-from opentelemetry import context as context_api-from opentelemetry import trace as trace_api-from opentelemetry.trace import Status, StatusCode--from atomic_agents import __version__-from atomic_agents.agents.atomic_agent import AtomicAgent-from atomic_agents.base.base_tool import BaseTool--if TYPE_CHECKING:- from opentelemetry.trace import TracerProvider--logger = logging.getLogger(__name__)--TRACED_MARKER = "__atomic_agents_traced__"-INHERITED_MARKER = "__atomic_agents_inherited__"-TRACER_NAME = "atomic_agents"--AGENT_SYNC_METHOD = "run"-AGENT_ASYNC_METHOD = "run_async"-AGENT_SYNC_STREAM_METHOD = "run_stream"-AGENT_ASYNC_STREAM_METHOD = "run_async_stream"-TOOL_METHOD_NAMES = ("run", "run_async", "arun")--ATTR_MODEL = "atomic_agents.model"-ATTR_MODE = "atomic_agents.mode"-ATTR_INPUT_SCHEMA = "atomic_agents.input_schema"-ATTR_OUTPUT_SCHEMA = "atomic_agents.output_schema"-ATTR_HISTORY_MESSAGE_COUNT = "atomic_agents.history.message_count"-ATTR_CONTEXT_PROVIDERS = "atomic_agents.context_providers"-ATTR_SYSTEM_PROMPT_SHA256 = "atomic_agents.system_prompt.sha256"-ATTR_STREAM_CLOSED_EARLY = "atomic_agents.stream.closed_early"--_active_tool: contextvars.ContextVar[Optional[BaseTool]] = contextvars.ContextVar("atomic_agents_active_tool", default=None)---class _Installation:- def __init__(- self,- tracer: OITracer,- tracer_provider: Optional["TracerProvider"],- config: TraceConfig,- original_init_subclass: classmethod,- ):- self.tracer = tracer- self.tracer_provider = tracer_provider- self.config = config- self.original_init_subclass = original_init_subclass---_installation: Optional[_Installation] = None---def is_installed() -> bool:- return _installation is not None---def install(*, tracer_provider: Optional["TracerProvider"] = None, config: Optional[TraceConfig] = None) -> None:- global _installation- config = config or TraceConfig()- if _installation is not None:- _warn_if_reinstalled_differently(_installation, tracer_provider, config)- return- tracer = OITracer(trace_api.get_tracer(TRACER_NAME, __version__, tracer_provider), config=config)- original_init_subclass = BaseTool.__dict__["__init_subclass__"]- _installation = _Installation(tracer, tracer_provider, config, original_init_subclass)- _instrument_agent_class(tracer)- _instrument_existing_tool_classes(tracer)- _hook_future_tool_classes(tracer, original_init_subclass)---def uninstall() -> None:- global _installation- if _installation is None:- return- setattr(BaseTool, "__init_subclass__", _installation.original_init_subclass)- for tool_class in _all_subclasses(BaseTool):- _restore_tool_class(tool_class)- for method_name in _AGENT_WRAPPERS:- _restore_method(AtomicAgent, method_name)- _installation = None---def _warn_if_reinstalled_differently(- installed: _Installation, tracer_provider: Optional["TracerProvider"], config: TraceConfig-) -> None:- same_provider = tracer_provider is None or tracer_provider is installed.tracer_provider- same_config = config == installed.config- if not (same_provider and same_config):- logger.warning(- "AtomicAgentsInstrumentor is already installed; the new tracer_provider/config are ignored. "- "Call uninstrument() first to change them."- )---def _instrument_agent_class(tracer: OITracer) -> None:- for method_name, wrap in _AGENT_WRAPPERS.items():- original = AtomicAgent.__dict__[method_name]- if not getattr(original, TRACED_MARKER, False):- setattr(AtomicAgent, method_name, _mark_traced(wrap(tracer, method_name, original), original, False))---def _mark_traced(wrapper: Callable, original: Callable, inherited: bool) -> Callable:- setattr(wrapper, TRACED_MARKER, True)- setattr(wrapper, INHERITED_MARKER, inherited)- setattr(wrapper, "__wrapped__", original)- return wrapper---def _restore_method(owner: Type, method_name: str) -> None:- current = owner.__dict__.get(method_name)- if current is None or not getattr(current, TRACED_MARKER, False):- return- if getattr(current, INHERITED_MARKER, False):- delattr(owner, method_name)- else:- setattr(owner, method_name, current.__wrapped__)---def _all_subclasses(base: Type) -> List[Type]:- found: List[Type] = []- pending = list(base.__subclasses__())- while pending:- current = pending.pop(0)- if current not in found:- found.append(current)- pending.extend(current.__subclasses__())- return found---def _instrument_existing_tool_classes(tracer: OITracer) -> None:- for tool_class in _all_subclasses(BaseTool):- _instrument_tool_class(tool_class, tracer)---def _hook_future_tool_classes(tracer: OITracer, original_init_subclass: classmethod) -> None:- def traced_init_subclass(cls, **kwargs):- original_init_subclass.__func__(cls, **kwargs)- _instrument_tool_class(cls, tracer)-- setattr(BaseTool, "__init_subclass__", classmethod(traced_init_subclass))---def _instrument_tool_class(tool_class: Type, tracer: OITracer) -> None:- for method_name in TOOL_METHOD_NAMES:- method = inspect.getattr_static(tool_class, method_name, None)- if _is_traceable_function(method):- inherited = method_name not in tool_class.__dict__- setattr(tool_class, method_name, _mark_traced(_trace_tool_method(tracer, method), method, inherited))---def _is_traceable_function(method: Any) -> bool:- if not isinstance(method, FunctionType) or inspect.isasyncgenfunction(method):- return False- if getattr(method, "__isabstractmethod__", False):- return False- return not getattr(method, TRACED_MARKER, False)---def _restore_tool_class(tool_class: Type) -> None:- for method_name in TOOL_METHOD_NAMES:- _restore_method(tool_class, method_name)---def _agent_display_name(agent: AtomicAgent) -> str:- return f"{type(agent).__name__}[{agent.input_schema.__name__}, {agent.output_schema.__name__}]"---def _agent_start_attributes(agent: AtomicAgent) -> Dict[str, Any]:- try:- return {- SpanAttributes.AGENT_NAME: _agent_display_name(agent),- ATTR_MODEL: agent.model,- ATTR_MODE: getattr(agent.mode, "value", str(agent.mode)),- ATTR_INPUT_SCHEMA: agent.input_schema.__name__,- ATTR_OUTPUT_SCHEMA: agent.output_schema.__name__,- ATTR_HISTORY_MESSAGE_COUNT: len(agent.history.history),- ATTR_CONTEXT_PROVIDERS: _context_provider_names(agent),- **_system_prompt_version(agent),- }- except Exception as error:- logger.warning("Could not describe agent for tracing: %s", error)- return {}---def _context_provider_names(agent: AtomicAgent) -> List[str]:- providers = getattr(agent.system_prompt_generator, "context_providers", None) or {}- return sorted(str(name) for name in providers)---def _system_prompt_version(agent: AtomicAgent) -> Dict[str, str]:- """Hashes only the static prompt sections, so the value identifies the prompt version across turns- even though context providers change the rendered prompt on every call."""- generator = agent.system_prompt_generator- sections = {name: getattr(generator, name, None) for name in ("background", "steps", "output_instructions")}- if agent.system_role is None or all(value is None for value in sections.values()):- return {}- digest = hashlib.sha256(json.dumps(sections, sort_keys=True, default=str).encode("utf-8")).hexdigest()- return {ATTR_SYSTEM_PROMPT_SHA256: digest}---def _user_input_from_call(args: tuple, kwargs: dict) -> Any:- return args[0] if args else kwargs.get("user_input")---def _set_input_safely(span: Any, value: Any) -> None:- if value is None:- return- try:- span.set_input(value)- except Exception as error:- logger.warning("Could not record span input: %s", error)---def _set_output_safely(span: Any, value: Any) -> None:- if value is None:- return- try:- span.set_output(value)- except Exception as error:- logger.warning("Could not record span output: %s", error)---def _finish_agent_span(span: Any, output: Any) -> None:- _set_output_safely(span, output)- span.set_status(Status(StatusCode.OK))---def _mark_closed_early(span: Any, output: Any) -> None:- span.set_attribute(ATTR_STREAM_CLOSED_EARLY, True)- _set_output_safely(span, output)---def _record_failure(span: Any, error: BaseException) -> None:- span.record_exception(error)- span.set_status(Status(StatusCode.ERROR, str(error)))---def _trace_agent_call(tracer: OITracer, method_name: str, wrapped: Callable) -> Callable:- @functools.wraps(wrapped)- def traced(agent, *args, **kwargs):- span_name = f"{_agent_display_name(agent)}.{method_name}"- with tracer.start_as_current_span(- span_name, openinference_span_kind="agent", attributes=_agent_start_attributes(agent)- ) as span:- _set_input_safely(span, _user_input_from_call(args, kwargs))- response = wrapped(agent, *args, **kwargs)- _finish_agent_span(span, response)- return response-- return traced---def _trace_agent_async_call(tracer: OITracer, method_name: str, wrapped: Callable) -> Callable:- @functools.wraps(wrapped)- async def traced(agent, *args, **kwargs):- span_name = f"{_agent_display_name(agent)}.{method_name}"- with tracer.start_as_current_span(- span_name, openinference_span_kind="agent", attributes=_agent_start_attributes(agent)- ) as span:- _set_input_safely(span, _user_input_from_call(args, kwargs))- response = await wrapped(agent, *args, **kwargs)- _finish_agent_span(span, response)- return response-- return traced---def _start_agent_span(tracer: OITracer, agent: AtomicAgent, method_name: str, args: tuple, kwargs: dict) -> Any:- span = tracer.start_span(- f"{_agent_display_name(agent)}.{method_name}",- openinference_span_kind="agent",- attributes=_agent_start_attributes(agent),- )- _set_input_safely(span, _user_input_from_call(args, kwargs))- return span---def _call_within_span(span: Any, function: Callable, *args: Any) -> Any:- token = context_api.attach(trace_api.set_span_in_context(span))- try:- return function(*args)- finally:- context_api.detach(token)---async def _await_within_span(span: Any, awaitable: Any) -> Any:- token = context_api.attach(trace_api.set_span_in_context(span))- try:- return await awaitable- finally:- context_api.detach(token)---class _SyncStreamCursor:- def __init__(self, span: Any, inner: Generator):- self.span = span- self.inner = inner- self.last: Any = None- self.final: Any = None-- def advance(self) -> bool:- try:- self.last = _call_within_span(self.span, next, self.inner)- except StopIteration as stop:- self.final = stop.value- return False- return True-- def close_early(self) -> None:- _call_within_span(self.span, self.inner.close)- _mark_closed_early(self.span, self.last)- self.span.set_status(Status(StatusCode.OK))---class _AsyncStreamCursor:- """Async twin of :class:`_SyncStreamCursor`; async generators carry no return value, so the last partial is the output."""-- def __init__(self, span: Any, inner: AsyncGenerator):- self.span = span- self.inner = inner- self.last: Any = None-- async def advance(self) -> bool:- try:- self.last = await _await_within_span(self.span, self.inner.__anext__())- except StopAsyncIteration:- return False- return True-- async def close_early(self) -> None:- await _await_within_span(self.span, self.inner.aclose())- _mark_closed_early(self.span, self.last)- self.span.set_status(Status(StatusCode.OK))-- def mark_cancelled(self) -> None:- _mark_closed_early(self.span, self.last)---def _trace_agent_stream(tracer: OITracer, method_name: str, wrapped: Callable) -> Callable:- @functools.wraps(wrapped)- def traced(agent, *args, **kwargs):- span = _start_agent_span(tracer, agent, method_name, args, kwargs)- cursor = _SyncStreamCursor(span, wrapped(agent, *args, **kwargs))- try:- while cursor.advance():- yield cursor.last- _finish_agent_span(span, cursor.final)- return cursor.final- except GeneratorExit:- cursor.close_early()- raise- except BaseException as error:- _record_failure(span, error)- raise- finally:- span.end()-- return traced---def _trace_agent_async_stream(tracer: OITracer, method_name: str, wrapped: Callable) -> Callable:- @functools.wraps(wrapped)- async def traced(agent, *args, **kwargs):- span = _start_agent_span(tracer, agent, method_name, args, kwargs)- cursor = _AsyncStreamCursor(span, wrapped(agent, *args, **kwargs))- try:- while await cursor.advance():- yield cursor.last- _finish_agent_span(span, cursor.last)- except GeneratorExit:- await cursor.close_early()- raise- except asyncio.CancelledError:- cursor.mark_cancelled()- raise- except BaseException as error:- _record_failure(span, error)- raise- finally:- span.end()-- return traced---def _describe_tool(span: Any, tool: BaseTool) -> None:- try:- span.set_tool(- name=tool.tool_name,- description=tool.tool_description,- parameters=tool.input_schema.model_json_schema(),- )- except Exception as error:- logger.warning("Could not describe tool %s for tracing: %s", type(tool).__name__, error)---def _tool_input_from_call(args: tuple, kwargs: dict) -> Any:- return args[0] if args else kwargs.get("params")---def _trace_tool_method(tracer: OITracer, wrapped: Callable) -> Callable:- if inspect.iscoroutinefunction(wrapped):- return _trace_tool_async_method(tracer, wrapped)-- @functools.wraps(wrapped)- def traced(tool, *args, **kwargs):- if _active_tool.get() is tool:- return wrapped(tool, *args, **kwargs)- with tracer.start_as_current_span(type(tool).__name__, openinference_span_kind="tool") as span:- _describe_tool(span, tool)- _set_input_safely(span, _tool_input_from_call(args, kwargs))- result = _run_as_active_tool(tool, wrapped, args, kwargs)- _set_output_safely(span, result)- span.set_status(Status(StatusCode.OK))- return result-- return traced---def _trace_tool_async_method(tracer: OITracer, wrapped: Callable) -> Callable:- @functools.wraps(wrapped)- async def traced(tool, *args, **kwargs):- if _active_tool.get() is tool:- return await wrapped(tool, *args, **kwargs)- with tracer.start_as_current_span(type(tool).__name__, openinference_span_kind="tool") as span:- _describe_tool(span, tool)- _set_input_safely(span, _tool_input_from_call(args, kwargs))- result = await _await_as_active_tool(tool, wrapped, args, kwargs)- _set_output_safely(span, result)- span.set_status(Status(StatusCode.OK))- return result-- return traced---def _run_as_active_tool(tool: BaseTool, wrapped: Callable, args: tuple, kwargs: dict) -> Any:- token = _active_tool.set(tool)- try:- return wrapped(tool, *args, **kwargs)- finally:- _active_tool.reset(token)---async def _await_as_active_tool(tool: BaseTool, wrapped: Callable, args: tuple, kwargs: dict) -> Any:- token = _active_tool.set(tool)- try:- return await wrapped(tool, *args, **kwargs)- finally:- _active_tool.reset(token)---_AGENT_WRAPPERS: Dict[str, Callable[[OITracer, str, Callable], Callable]] = {- AGENT_SYNC_METHOD: _trace_agent_call,- AGENT_ASYNC_METHOD: _trace_agent_async_call,- AGENT_SYNC_STREAM_METHOD: _trace_agent_stream,- AGENT_ASYNC_STREAM_METHOD: _trace_agent_async_stream,-}diff --git a/atomic-agents/atomic_agents/observability/span_attributes.py b/atomic-agents/atomic_agents/observability/span_attributes.pynew file mode 100644index 0000000..ccbf615--- /dev/null+++ b/atomic-agents/atomic_agents/observability/span_attributes.py@@ -0,0 +1,190 @@+"""Span attributes and conversation context."""++import contextvars+import hashlib+import json+import logging+from collections.abc import Mapping+from contextlib import contextmanager+from typing import Callable, Dict, Iterator, List++from opentelemetry import context as context_api+from opentelemetry.util.types import AttributeValue++from atomic_agents.agents.atomic_agent import AtomicAgent++from . import installation+++logger = logging.getLogger("atomic_agents.observability.tracing")+++ATTR_MODEL = "atomic_agents.model"+ATTR_MODE = "atomic_agents.mode"+ATTR_INPUT_SCHEMA = "atomic_agents.input_schema"+ATTR_OUTPUT_SCHEMA = "atomic_agents.output_schema"+ATTR_HISTORY_MESSAGE_COUNT = "atomic_agents.history.message_count"+ATTR_CONTEXT_PROVIDERS = "atomic_agents.context_providers"+ATTR_SYSTEM_PROMPT_SHA256 = "atomic_agents.system_prompt.sha256"+ATTR_STREAM_CLOSED_EARLY = "atomic_agents.stream.closed_early"+ATTR_OPERATION = "gen_ai.operation.name"+ATTR_AGENT_NAME = "gen_ai.agent.name"+ATTR_REQUEST_MODEL = "gen_ai.request.model"+ATTR_PROVIDER_NAME = "gen_ai.provider.name"+ATTR_CONVERSATION_ID = "gen_ai.conversation.id"+ATTR_TOOL_NAME = "gen_ai.tool.name"+ATTR_TOOL_DESCRIPTION = "gen_ai.tool.description"+ATTR_TOOL_TYPE = "gen_ai.tool.type"+ATTR_TOOL_ARGUMENTS = "gen_ai.tool.call.arguments"+ATTR_TOOL_RESULT = "gen_ai.tool.call.result"+ATTR_SPAN_KIND = "openinference.span.kind"+ATTR_INPUT_VALUE = "input.value"+ATTR_INPUT_MIME_TYPE = "input.mime_type"+ATTR_OUTPUT_VALUE = "output.value"+ATTR_OUTPUT_MIME_TYPE = "output.mime_type"+ATTR_OI_AGENT_NAME = "agent.name"+ATTR_OI_TOOL_NAME = "tool.name"+ATTR_OI_TOOL_DESCRIPTION = "tool.description"+ATTR_TOOL_PARAMETERS = "tool.parameters"+ATTR_SESSION_ID = "session.id"+ATTR_INPUT = "atomic_agents.input"+ATTR_OUTPUT = "atomic_agents.output"+ATTR_AGENT_CLASS = "atomic_agents.agent.class"+ATTR_ERROR_TYPE = "error.type"+ATTR_TOOL_CLASS = "atomic_agents.tool.class"+PROVIDER_NAMES = {+ "azure_openai": "azure.ai.openai",+ "vertexai": "gcp.vertex_ai",+ "gemini": "gcp.gemini",+ "genai": "gcp.gemini",+ "bedrock": "aws.bedrock",+ "mistral": "mistral_ai",+ "xai": "x_ai",+}+++def _import_context_attributes() -> Callable[[], dict[str, AttributeValue]] | None:+ try:+ from openinference.instrumentation import get_attributes_from_context+ except ImportError:+ return None+ from atomic_agents.observability import _openinference_is_too_old++ if _openinference_is_too_old():+ logger.warning("OpenInference context attributes disabled: 0.1.38 or newer is required.")+ return None+ return lambda: dict(get_attributes_from_context())+++def _context_attributes() -> dict[str, AttributeValue]:+ reader = installation._current_installation().context_attributes+ return reader() if reader is not None else {}+++def _suppressed() -> bool:+ return bool(context_api.get_value("suppress_instrumentation"))+++_conversation_id: contextvars.ContextVar[str | None] = contextvars.ContextVar("atomic_agents_conversation_id", default=None)+++@contextmanager+def using_conversation(conversation_id: str) -> Iterator[None]:+ token = _conversation_id.set(conversation_id)+ try:+ yield+ finally:+ _conversation_id.reset(token)+++def _agent_display_name(agent: AtomicAgent) -> str:+ attributes = _best_effort_attributes({ATTR_AGENT_NAME: lambda: getattr(agent, "name", None)})+ return str(attributes.get(ATTR_AGENT_NAME) or type(agent).__name__)+++def _agent_class_name(agent: AtomicAgent) -> str:+ return f"{type(agent).__name__}[{agent.input_schema.__name__}, {agent.output_schema.__name__}]"+++def _best_effort_attributes(+ readers: dict[str, Callable[[], AttributeValue | None]],+) -> dict[str, AttributeValue]:+ attributes: dict[str, AttributeValue] = {}+ for key, read in readers.items():+ try:+ value = read()+ if value is not None:+ attributes[key] = value+ except Exception as error:+ logger.debug("Could not collect tracing attribute %s: %s", key, error)+ return attributes+++def _without_none(attributes: Mapping[str, AttributeValue | None]) -> dict[str, AttributeValue]:+ return {key: value for key, value in attributes.items() if value is not None}+++def _agent_start_attributes(agent: AtomicAgent) -> dict[str, AttributeValue]:+ attributes: dict[str, AttributeValue] = {+ ATTR_OPERATION: "invoke_agent",+ ATTR_AGENT_NAME: _agent_display_name(agent),+ ATTR_AGENT_CLASS: _agent_class_name(agent),+ }+ attributes.update(+ _best_effort_attributes(+ {+ ATTR_REQUEST_MODEL: lambda: agent.model,+ ATTR_MODEL: lambda: agent.model,+ ATTR_MODE: lambda: getattr(agent.mode, "value", None),+ ATTR_INPUT_SCHEMA: lambda: agent.input_schema.__name__,+ ATTR_OUTPUT_SCHEMA: lambda: agent.output_schema.__name__,+ ATTR_HISTORY_MESSAGE_COUNT: lambda: len(agent.history.history),+ ATTR_CONTEXT_PROVIDERS: lambda: _context_provider_names(agent),+ ATTR_SYSTEM_PROMPT_SHA256: lambda: _system_prompt_version(agent).get(ATTR_SYSTEM_PROMPT_SHA256),+ ATTR_PROVIDER_NAME: lambda: _provider_attributes(agent).get(ATTR_PROVIDER_NAME),+ }+ )+ )+ attributes.update(_optional_attributes("AGENT", {ATTR_OI_AGENT_NAME: _agent_display_name(agent)}))+ return _without_none(attributes)+++def _context_provider_names(agent: AtomicAgent) -> List[str]:+ providers = getattr(agent.system_prompt_generator, "context_providers", None) or {}+ return sorted(str(name) for name in providers)+++def _system_prompt_version(agent: AtomicAgent) -> Dict[str, str]:+ """Hashes only the static prompt sections, so the value identifies the prompt version across turns+ even though context providers change the rendered prompt on every call."""+ generator = agent.system_prompt_generator+ sections = {name: getattr(generator, name, None) for name in ("background", "steps", "output_instructions")}+ if agent.system_role is None or all(value is None for value in sections.values()):+ return {}+ digest = hashlib.sha256(json.dumps(sections, sort_keys=True, default=str).encode("utf-8")).hexdigest()+ return {ATTR_SYSTEM_PROMPT_SHA256: digest}+++def _provider_attributes(agent: AtomicAgent) -> dict[str, str]:+ provider = getattr(getattr(agent.client, "provider", None), "value", None)+ return {ATTR_PROVIDER_NAME: PROVIDER_NAMES.get(provider, provider)} if isinstance(provider, str) else {}+++def _optional_attributes(kind: str, attributes: dict[str, AttributeValue]) -> dict[str, AttributeValue]:+ context = _context_attributes()+ conversation = _conversation_attributes(context)+ if not installation._current_installation().config.openinference_attributes:+ return conversation+ return _without_none({ATTR_SPAN_KIND: kind, **attributes, **context, **conversation})+++def _conversation_attributes(context: dict[str, AttributeValue]) -> dict[str, AttributeValue]:+ conversation_id: AttributeValue | None = _conversation_id.get()+ if conversation_id is None:+ conversation_id = context.get(ATTR_SESSION_ID)+ if conversation_id is None:+ return {}+ attributes = {ATTR_CONVERSATION_ID: conversation_id}+ if installation._current_installation().config.openinference_attributes:+ attributes[ATTR_SESSION_ID] = conversation_id+ return attributesdiff --git a/atomic-agents/atomic_agents/observability/span_payloads.py b/atomic-agents/atomic_agents/observability/span_payloads.pynew file mode 100644index 0000000..62317f2--- /dev/null+++ b/atomic-agents/atomic_agents/observability/span_payloads.py@@ -0,0 +1,143 @@+"""Payload serialization, redaction, and bounded failures."""++import json+import logging+import traceback+from collections.abc import Iterator++from instructor.core.exceptions import InstructorError++from opentelemetry.trace import Span, Status, StatusCode+from pydantic import BaseModel, ValidationError++from . import installation+from .tracing_config import TracingConfig+from .span_attributes import (+ ATTR_INPUT_VALUE,+ ATTR_INPUT_MIME_TYPE,+ ATTR_OUTPUT_VALUE,+ ATTR_OUTPUT_MIME_TYPE,+ ATTR_INPUT,+ ATTR_OUTPUT,+ ATTR_ERROR_TYPE,+)+++logger = logging.getLogger("atomic_agents.observability.tracing")+++TRUNCATION_MARKER = "...[truncated]"+++def _serialize_payload(value: object) -> str | None:+ try:+ return value.model_dump_json() if isinstance(value, BaseModel) else json.dumps(value, default=str)+ except Exception as error:+ logger.warning("Could not serialize tracing payload: %s", error)+ return None+++def _record_payload(+ span: Span, key_pairs: tuple[tuple[str, str | None], ...], value: object, hide: bool, config: TracingConfig+) -> None:+ if value is None:+ return+ serialized = "__REDACTED__" if hide else _serialize_payload(value)+ if serialized is None:+ return+ for key, mime_key in key_pairs:+ _write_payload(span, key, mime_key, serialized, hide, config)+++def _write_payload(span: Span, key: str, mime_key: str | None, serialized: str, hide: bool, config: TracingConfig) -> None:+ if hide:+ span.set_attribute(key, "__REDACTED__")+ return+ if len(serialized) > config.max_attribute_length:+ span.set_attribute(key + ".original_length", len(serialized))+ serialized = _bounded_text(serialized, config)+ span.set_attribute(key, serialized)+ if mime_key is not None:+ span.set_attribute(mime_key, "application/json")+++def _payload_keys(primary: str, output: bool) -> tuple[tuple[str, str | None], ...]:+ keys: tuple[tuple[str, str | None], ...] = ((primary, None),)+ if installation._current_installation().config.openinference_attributes:+ keys += ((ATTR_OUTPUT_VALUE, ATTR_OUTPUT_MIME_TYPE),) if output else ((ATTR_INPUT_VALUE, ATTR_INPUT_MIME_TYPE),)+ return keys+++def _set_input_safely(span: Span, value: object, primary: str = ATTR_INPUT) -> None:+ config = installation._current_installation().config+ _record_payload(span, _payload_keys(primary, False), value, config.hide_inputs, config)+++def _set_output_safely(span: Span, value: object, primary: str = ATTR_OUTPUT) -> None:+ config = installation._current_installation().config+ _record_payload(span, _payload_keys(primary, True), value, config.hide_outputs, config)+++def _bounded_text(value: str, config: TracingConfig) -> str:+ limit = config.max_attribute_length+ if len(value) > limit:+ marker = TRUNCATION_MARKER[:limit]+ return value[: limit - len(marker)] + marker+ return value+++def _exception_chain(error: BaseException) -> Iterator[BaseException]:+ seen: set[int] = set()+ while id(error) not in seen:+ seen.add(id(error))+ yield error+ following = error.__cause__ or error.__context__+ if following is None:+ return+ error = following+++def _instructor_attempt_count(error: InstructorError) -> int:+ if error.failed_attempts:+ return len(error.failed_attempts)+ return getattr(error, "n_attempts", 0)+++def _failure_count(error: BaseException) -> str | None:+ if isinstance(error, InstructorError):+ return f"{_instructor_attempt_count(error)} attempts"+ if isinstance(error, ValidationError):+ return f"{error.error_count()} errors"+ return None+++def _failure_message(error: BaseException, config: TracingConfig) -> str:+ if not config.hide_outputs:+ return str(error)+ count = next((count for cause in _exception_chain(error) if (count := _failure_count(cause)) is not None), "1 errors")+ return f"{type(error).__qualname__}: {count} (__REDACTED__)"+++def _failure_stacktrace(error: BaseException, config: TracingConfig) -> str:+ lines = traceback.format_exception(error, chain=not config.hide_outputs)+ if config.hide_outputs:+ lines = _redacted_traceback(error, config)+ return _bounded_text("".join(lines), config)+++def _redacted_traceback(error: BaseException, config: TracingConfig) -> list[str]:+ frames = [+ traceback.FrameSummary(frame.filename, frame.lineno, frame.name, line="")+ for frame in traceback.extract_tb(error.__traceback__)+ ]+ return ["Traceback (most recent call last):\n", *traceback.format_list(frames), _failure_message(error, config) + "\n"]+++def _record_failure(span: Span, error: BaseException, config: TracingConfig | None = None) -> None:+ config = config or installation._current_installation().config+ message = _bounded_text(_failure_message(error, config), config)+ span.set_attribute(ATTR_ERROR_TYPE, type(error).__qualname__)+ span.record_exception(+ error, attributes={"exception.message": message, "exception.stacktrace": _failure_stacktrace(error, config)}+ )+ span.set_status(Status(StatusCode.ERROR, message))diff --git a/atomic-agents/atomic_agents/observability/tool_spans.py b/atomic-agents/atomic_agents/observability/tool_spans.pynew file mode 100644index 0000000..7a140aa--- /dev/null+++ b/atomic-agents/atomic_agents/observability/tool_spans.py@@ -0,0 +1,208 @@+"""Tool spans and tool class instrumentation."""++import asyncio+import contextvars+import functools+import inspect+import json+import logging+from contextlib import contextmanager+from types import FunctionType+from typing import Awaitable, Callable, Coroutine, Iterator, List, Optional, Type, TypeVar++from opentelemetry.trace import Span, Status, StatusCode+from opentelemetry.util.types import AttributeValue++from atomic_agents.base.base_tool import BaseTool++from . import installation+from .span_attributes import _best_effort_attributes, _optional_attributes, _suppressed, _without_none+from .span_payloads import _bounded_text, _record_failure, _set_input_safely, _set_output_safely+from .span_attributes import (+ ATTR_OPERATION,+ ATTR_TOOL_NAME,+ ATTR_TOOL_DESCRIPTION,+ ATTR_TOOL_TYPE,+ ATTR_TOOL_ARGUMENTS,+ ATTR_TOOL_RESULT,+ ATTR_OI_TOOL_NAME,+ ATTR_OI_TOOL_DESCRIPTION,+ ATTR_TOOL_PARAMETERS,+ ATTR_TOOL_CLASS,+)+++logger = logging.getLogger("atomic_agents.observability.tracing")+++OutputSchema = TypeVar("OutputSchema")++TOOL_METHOD_NAMES = ("run", "run_async", "arun")+++_active_tool: contextvars.ContextVar[Optional[BaseTool]] = contextvars.ContextVar("atomic_agents_active_tool", default=None)+++def _all_subclasses(base: Type) -> List[Type]:+ found: List[Type] = []+ pending = list(base.__subclasses__())+ while pending:+ current = pending.pop(0)+ if current not in found:+ found.append(current)+ pending.extend(current.__subclasses__())+ return found+++def _instrument_existing_tool_classes() -> None:+ for tool_class in _all_subclasses(BaseTool):+ _instrument_tool_class(tool_class)+++def _hook_future_tool_classes(original_init_subclass: classmethod) -> None:+ def traced_init_subclass(cls, **kwargs):+ original_init_subclass.__func__(cls, **kwargs)+ _instrument_tool_class(cls)++ setattr(BaseTool, "__init_subclass__", classmethod(traced_init_subclass))+++def _instrument_tool_class(tool_class: Type) -> None:+ for method_name in TOOL_METHOD_NAMES:+ method = inspect.getattr_static(tool_class, method_name, None)+ if _is_traceable_function(method):+ inherited = method_name not in tool_class.__dict__+ setattr(tool_class, method_name, installation._mark_traced(_trace_tool_method(method), method, inherited))+++def _is_traceable_function(method: object) -> bool:+ if not isinstance(method, FunctionType) or inspect.isasyncgenfunction(method):+ return False+ if getattr(method, "__isabstractmethod__", False):+ return False+ return not getattr(method, installation.TRACED_MARKER, False)+++def _restore_tool_class(tool_class: Type) -> None:+ for method_name in TOOL_METHOD_NAMES:+ installation._restore_method(tool_class, method_name)+++def _tool_parameter_attributes(tool: BaseTool) -> dict[str, AttributeValue]:+ parameters = _best_effort_attributes({ATTR_TOOL_PARAMETERS: lambda: json.dumps(tool.input_schema.model_json_schema())})+ value = parameters.get(ATTR_TOOL_PARAMETERS)+ if isinstance(value, str):+ config = installation._current_installation().config+ parameters[ATTR_TOOL_PARAMETERS] = _bounded_text(value, config)+ if len(value) > config.max_attribute_length:+ parameters[ATTR_TOOL_PARAMETERS + ".original_length"] = len(value)+ return parameters+++def _tool_display_name(tool: BaseTool) -> str:+ try:+ return tool.tool_name+ except Exception as error:+ logger.debug("Could not collect tracing tool name: %s", error)+ return type(tool).__name__+++def _tool_start_attributes(tool: BaseTool) -> dict[str, AttributeValue]:+ attributes: dict[str, AttributeValue] = {+ ATTR_OPERATION: "execute_tool",+ ATTR_TOOL_NAME: _tool_display_name(tool),+ ATTR_TOOL_CLASS: type(tool).__name__,+ ATTR_TOOL_TYPE: "function",+ **_optional_attributes("TOOL", {}),+ }+ attributes.update(_best_effort_attributes({ATTR_TOOL_DESCRIPTION: lambda: tool.tool_description}))+ if installation._current_installation().config.openinference_attributes:+ attributes.update(+ _best_effort_attributes(+ {+ ATTR_OI_TOOL_NAME: lambda: attributes[ATTR_TOOL_NAME],+ ATTR_OI_TOOL_DESCRIPTION: lambda: tool.tool_description,+ }+ )+ )+ attributes.update(_tool_parameter_attributes(tool))+ return _without_none(attributes)+++@contextmanager+def _tool_span(tool: BaseTool, args: tuple[object, ...], kwargs: dict[str, object]) -> Iterator[Span]:+ attributes = _tool_start_attributes(tool)+ with installation._current_installation().tracer.start_as_current_span(+ f"execute_tool {attributes[ATTR_TOOL_NAME]}",+ attributes=attributes,+ record_exception=False,+ set_status_on_exception=False,+ ) as span:+ _set_input_safely(span, _tool_input_from_call(args, kwargs), ATTR_TOOL_ARGUMENTS)+ try:+ yield span+ except (asyncio.CancelledError, GeneratorExit):+ span.set_attribute("atomic_agents.cancelled", True)+ raise+ except BaseException as error:+ _record_failure(span, error)+ raise+++def _tool_input_from_call(args: tuple[object, ...], kwargs: dict[str, object]) -> object:+ return args[0] if args else kwargs.get("params")+++def _trace_tool_method(+ wrapped: Callable[..., OutputSchema],+) -> Callable[..., OutputSchema] | Callable[..., Coroutine[object, object, object]]:+ if inspect.iscoroutinefunction(wrapped):+ return _trace_tool_async_method(wrapped)++ @functools.wraps(wrapped)+ def traced(tool: BaseTool, *args: object, **kwargs: object) -> OutputSchema:+ if _active_tool.get() is tool or _suppressed():+ return wrapped(tool, *args, **kwargs)+ with _tool_span(tool, args, kwargs) as span:+ result = _run_as_active_tool(tool, wrapped, args, kwargs)+ _set_output_safely(span, result, ATTR_TOOL_RESULT)+ span.set_status(Status(StatusCode.OK))+ return result++ return traced+++def _trace_tool_async_method(+ wrapped: Callable[..., Awaitable[OutputSchema]],+) -> Callable[..., Coroutine[object, object, OutputSchema]]:+ @functools.wraps(wrapped)+ async def traced(tool: BaseTool, *args: object, **kwargs: object) -> OutputSchema:+ if _active_tool.get() is tool or _suppressed():+ return await wrapped(tool, *args, **kwargs)+ with _tool_span(tool, args, kwargs) as span:+ result = await _await_as_active_tool(tool, wrapped, args, kwargs)+ _set_output_safely(span, result, ATTR_TOOL_RESULT)+ span.set_status(Status(StatusCode.OK))+ return result++ return traced+++def _run_as_active_tool(+ tool: BaseTool, wrapped: Callable[..., OutputSchema], args: tuple[object, ...], kwargs: dict[str, object]+) -> OutputSchema:+ token = _active_tool.set(tool)+ try:+ return wrapped(tool, *args, **kwargs)+ finally:+ _active_tool.reset(token)+++async def _await_as_active_tool(+ tool: BaseTool, wrapped: Callable[..., Awaitable[OutputSchema]], args: tuple[object, ...], kwargs: dict[str, object]+) -> OutputSchema:+ token = _active_tool.set(tool)+ try:+ return await wrapped(tool, *args, **kwargs)+ finally:+ _active_tool.reset(token)diff --git a/atomic-agents/atomic_agents/observability/tracing.py b/atomic-agents/atomic_agents/observability/tracing.pynew file mode 100644index 0000000..4cb43e5--- /dev/null+++ b/atomic-agents/atomic_agents/observability/tracing.py@@ -0,0 +1,11 @@+"""Compatibility exports for Atomic Agents tracing."""++from . import installation, span_attributes, tracing_config++TRACER_NAME = installation.TRACER_NAME+install = installation.install+is_installed = installation.is_installed+uninstall = installation.uninstall+PayloadVisibility = tracing_config.PayloadVisibility+TracingConfig = tracing_config.TracingConfig+using_conversation = span_attributes.using_conversationdiff --git a/atomic-agents/atomic_agents/observability/tracing_config.py b/atomic-agents/atomic_agents/observability/tracing_config.pynew file mode 100644index 0000000..be16d88--- /dev/null+++ b/atomic-agents/atomic_agents/observability/tracing_config.py@@ -0,0 +1,57 @@+"""Tracing configuration and payload visibility."""++import logging+import os+from dataclasses import dataclass, replace+from typing import Protocol+++logger = logging.getLogger("atomic_agents.observability.tracing")+DEFAULT_MAX_ATTRIBUTE_LENGTH = 4096+++class PayloadVisibility(Protocol):+ hide_inputs: bool+ hide_outputs: bool+++@dataclass(frozen=True)+class TracingConfig:+ hide_inputs: bool = False+ hide_outputs: bool = False+ max_attribute_length: int = DEFAULT_MAX_ATTRIBUTE_LENGTH+ openinference_attributes: bool | None = None+ llm_spans: bool = True++ @classmethod+ def from_environment(cls) -> "TracingConfig":+ return cls(+ hide_inputs=_visibility_environment("INPUTS").lower() in ("true", "1"),+ hide_outputs=_visibility_environment("OUTPUTS").lower() in ("true", "1"),+ max_attribute_length=_attribute_length_environment(),+ llm_spans=os.environ.get("ATOMIC_AGENTS_TRACE_LLM_SPANS", "true").lower() not in ("false", "0"),+ )++ @classmethod+ def coerce(cls, config: "TracingConfig | PayloadVisibility | None") -> "TracingConfig":+ if config is None:+ return cls.from_environment()+ if isinstance(config, cls):+ return config+ return replace(cls.from_environment(), hide_inputs=config.hide_inputs, hide_outputs=config.hide_outputs)+++def _visibility_environment(direction: str) -> str:+ return os.environ.get(f"ATOMIC_AGENTS_TRACE_HIDE_{direction}", os.environ.get(f"OPENINFERENCE_HIDE_{direction}", ""))+++def _attribute_length_environment() -> int:+ value = os.environ.get("ATOMIC_AGENTS_TRACE_MAX_ATTRIBUTE_LENGTH", str(DEFAULT_MAX_ATTRIBUTE_LENGTH))+ try:+ length = int(value)+ if length >= 1:+ return length+ except ValueError:+ pass+ logger.warning("Invalid ATOMIC_AGENTS_TRACE_MAX_ATTRIBUTE_LENGTH value %r; using %d", value, DEFAULT_MAX_ATTRIBUTE_LENGTH)+ return DEFAULT_MAX_ATTRIBUTE_LENGTHdiff --git a/atomic-agents/tests/agents/test_agent_config_name.py b/atomic-agents/tests/agents/test_agent_config_name.pynew file mode 100644index 0000000..707e339--- /dev/null+++ b/atomic-agents/tests/agents/test_agent_config_name.py@@ -0,0 +1,22 @@+import instructor+from openai import OpenAI++from atomic_agents import AgentConfig, AtomicAgent, BaseIOSchema+++def test_agent_name_defaults_to_none() -> None:+ with OpenAI(api_key="synthetic-test-key") as provider_client:+ client = instructor.from_openai(provider_client)+ configuration = AgentConfig(client=client)+ agent = AtomicAgent[BaseIOSchema, BaseIOSchema](configuration)+ assert configuration.name is None+ assert agent.name is None+++def test_explicit_agent_name_is_stored() -> None:+ with OpenAI(api_key="synthetic-test-key") as provider_client:+ client = instructor.from_openai(provider_client)+ configuration = AgentConfig(client=client, name="orchestrator_planning")+ agent = AtomicAgent[BaseIOSchema, BaseIOSchema](configuration)+ assert configuration.name == "orchestrator_planning"+ assert agent.name == "orchestrator_planning"diff --git a/atomic-agents/tests/agents/test_agent_hooks.py b/atomic-agents/tests/agents/test_agent_hooks.pynew file mode 100644index 0000000..0afedfb--- /dev/null+++ b/atomic-agents/tests/agents/test_agent_hooks.py@@ -0,0 +1,26 @@+import instructor+import pytest+from unittest.mock import Mock++from atomic_agents import AgentConfig, AtomicAgent++[email protected]("event", [None, "completion:kwargs"])+def test_clear_hooks_preserves_foreign_handlers(event):+ client = instructor.Instructor(client=Mock(), create=Mock())+ agent = AtomicAgent(AgentConfig(client=client))+ foreign = Mock()+ owned = Mock()+ other_owned = Mock()+ client.on("completion:kwargs", foreign)+ agent.register_hook("completion:kwargs", owned)+ agent.register_hook("completion:response", other_owned)++ agent.clear_hooks(event)+ client.hooks.emit_completion_arguments(model="test-model")+ client.hooks.emit_completion_response(Mock())++ foreign.assert_called_once_with(model="test-model")+ owned.assert_not_called()+ assert other_owned.call_count == (0 if event is None else 1)+ assert agent._hook_handlers.get("completion:kwargs", []) == []diff --git a/atomic-agents/tests/observability/instructor_transport.py b/atomic-agents/tests/observability/instructor_transport.pynew file mode 100644index 0000000..c1d2a9a--- /dev/null+++ b/atomic-agents/tests/observability/instructor_transport.py@@ -0,0 +1,104 @@+"""Offline Instructor clients using real OpenAI request and validation paths."""++from dataclasses import dataclass++import httpx+import instructor+import openai+from openai.types import CompletionUsage+from openai.types.completion_usage import CompletionTokensDetails, PromptTokensDetails+++@dataclass(frozen=True)+class ScriptedResponse:+ arguments: str = '{"answer": "42"}'+ input_tokens: int = 3+ output_tokens: int = 2+ cached_tokens: int = 1+ reasoning_tokens: int = 2+ schema_name: str = "Answer"++ def chat_usage(self) -> CompletionUsage:+ return CompletionUsage(+ prompt_tokens=self.input_tokens,+ completion_tokens=self.output_tokens,+ total_tokens=self.input_tokens + self.output_tokens,+ prompt_tokens_details=PromptTokensDetails(cached_tokens=self.cached_tokens),+ completion_tokens_details=CompletionTokensDetails(reasoning_tokens=self.reasoning_tokens),+ )++ def responses_body(self) -> dict[str, object]:+ return {+ "id": "resp_test",+ "object": "response",+ "created_at": 1,+ "status": "completed",+ "model": "response-model",+ "output": [+ {+ "type": "function_call",+ "id": "fc_test",+ "call_id": "call_test",+ "name": self.schema_name,+ "arguments": self.arguments,+ "status": "completed",+ }+ ],+ "usage": {+ "input_tokens": self.input_tokens,+ "output_tokens": self.output_tokens,+ "total_tokens": self.input_tokens + self.output_tokens,+ "input_tokens_details": {"cached_tokens": self.cached_tokens},+ "output_tokens_details": {"reasoning_tokens": self.reasoning_tokens},+ },+ }++ def chat_body(self, mode: instructor.Mode) -> dict[str, object]:+ message: dict[str, object] = {"role": "assistant", "content": self.arguments}+ if mode == instructor.Mode.TOOLS:+ message = {+ "role": "assistant",+ "content": None,+ "tool_calls": [+ {+ "id": "call_test",+ "type": "function",+ "function": {"name": self.schema_name, "arguments": self.arguments},+ }+ ],+ }+ return {+ "id": "chatcmpl_test",+ "object": "chat.completion",+ "created": 1,+ "model": "response-model",+ "choices": [{"index": 0, "message": message, "finish_reason": "stop"}],+ "usage": self.chat_usage().model_dump(),+ }+++def response_transport(mode: instructor.Mode, responses: list[ScriptedResponse]) -> httpx.MockTransport:+ scripted = iter(responses)++ def respond(request: httpx.Request) -> httpx.Response:+ response = next(scripted)+ body = response.responses_body() if mode.value == "responses_tools" else response.chat_body(mode)+ expected_path = "/v1/responses" if mode.value == "responses_tools" else "/v1/chat/completions"+ if request.url.path != expected_path:+ raise AssertionError(f"Unexpected transport path: {request.url.path}")+ return httpx.Response(200, json=body)++ return httpx.MockTransport(respond)+++def instructor_client(mode: instructor.Mode, responses: list[ScriptedResponse]) -> instructor.Instructor:+ return instructor.from_openai(+ openai.OpenAI(api_key="test", http_client=httpx.Client(transport=response_transport(mode, responses))), mode=mode+ )+++def async_instructor_client(mode: instructor.Mode, responses: list[ScriptedResponse]) -> instructor.AsyncInstructor:+ return instructor.from_openai(+ openai.AsyncOpenAI(api_key="test", http_client=httpx.AsyncClient(transport=response_transport(mode, responses))),+ mode=mode,+ )diff --git a/atomic-agents/tests/observability/test_instructor_modes.py b/atomic-agents/tests/observability/test_instructor_modes.pynew file mode 100644index 0000000..7c2e4dc--- /dev/null+++ b/atomic-agents/tests/observability/test_instructor_modes.py@@ -0,0 +1,141 @@+"""Real Instructor transport, validation and tracing across Ada's request modes."""++import asyncio++import instructor+import pytest+from instructor.core.exceptions import InstructorRetryException+from opentelemetry.sdk.trace import TracerProvider+from opentelemetry.sdk.trace.export import SimpleSpanProcessor+from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter+from opentelemetry.trace import SpanKind, StatusCode++from instructor_transport import ScriptedResponse, async_instructor_client, instructor_client++from atomic_agents import AgentConfig, AtomicAgent, BaseIOSchema+from atomic_agents.observability import TracingConfig+from atomic_agents.observability import installation+++class Question(BaseIOSchema):+ """A question sent through the real client."""++ question: str+++class Answer(BaseIOSchema):+ """The validated model answer."""++ answer: str++[email protected](params=["TOOLS", "JSON", "JSON_SCHEMA", "RESPONSES_TOOLS"])+def mode(request):+ selected = getattr(instructor.Mode, request.param, None)+ if selected is None:+ pytest.skip(f"Installed Instructor has no Mode.{request.param} for from_openai")+ return selected++[email protected](params=[False, True], ids=["run", "run_async"])+def asynchronous(request):+ return request.param+++def exporter():+ exported = InMemorySpanExporter()+ provider = TracerProvider()+ provider.add_span_processor(SimpleSpanProcessor(exported))+ installation.install(tracer_provider=provider, config=TracingConfig(openinference_attributes=False))+ yield exported+ installation.uninstall()+ provider.shutdown()+++def run_script(mode, asynchronous, responses):+ construct_client = async_instructor_client if asynchronous else instructor_client+ client = construct_client(mode, responses)+ configured = AtomicAgent[Question, Answer](AgentConfig(client=client, model="test-model"))+ configured.model_api_parameters["max_retries"] = 1+ question = Question(question="What is the answer?")+ if asynchronous:+ return asyncio.run(configured.run_async(question))+ return configured.run(question)+++def assert_usage(span, input_tokens, output_tokens, cached_tokens, reasoning_tokens):+ assert span.attributes["gen_ai.usage.input_tokens"] == input_tokens+ assert span.attributes["gen_ai.usage.output_tokens"] == output_tokens+ assert span.attributes["gen_ai.usage.cache_read.input_tokens"] == cached_tokens+ assert span.attributes["gen_ai.usage.reasoning.output_tokens"] == reasoning_tokens+++def assert_attempt_spans(exporter, responses):+ spans = exporter.get_finished_spans()+ parent = spans[-1]+ attempts = spans[:-1]+ assert parent.attributes["gen_ai.operation.name"] == "invoke_agent"+ assert len(attempts) == len(responses)+ assert len(spans) == len(responses) + 1+ assert parent.attributes["atomic_agents.llm.attempts"] == len(responses)+ assert_models(parent)+ for number, (attempt, response) in enumerate(zip(attempts, responses), start=1):+ assert attempt.attributes["gen_ai.operation.name"] == "chat"+ assert attempt.name == "chat test-model"+ assert attempt.kind == SpanKind.CLIENT+ assert attempt.parent.span_id == parent.context.span_id+ assert attempt.attributes["atomic_agents.llm.attempt"] == number+ assert_models(attempt)+ assert_usage(attempt, response.input_tokens, response.output_tokens, response.cached_tokens, response.reasoning_tokens)+ return attempts, parent+++def assert_models(span):+ assert span.attributes["gen_ai.request.model"] == "test-model"+ assert span.attributes["gen_ai.response.model"] == "response-model"+ assert span.attributes["gen_ai.provider.name"] == "openai"+++def test_success_model_usage_and_attempt_count(mode, asynchronous, exporter):+ responses = [ScriptedResponse()]+ assert run_script(mode, asynchronous, responses).answer == "42"+ attempts, parent = assert_attempt_spans(exporter, responses)+ assert attempts[0].status.status_code == parent.status.status_code == StatusCode.OK+ assert_usage(parent, 3, 2, 1, 2)+++def test_validation_retry_preserves_each_attempt_and_total_usage(mode, asynchronous, exporter):+ responses = [+ ScriptedResponse(arguments='{"answer": {"secret": "private output"}}'),+ ScriptedResponse(input_tokens=7, output_tokens=5, cached_tokens=3, reasoning_tokens=4),+ ]+ assert run_script(mode, asynchronous, responses).answer == "42"+ attempts, parent = assert_attempt_spans(exporter, responses)+ assert [attempt.status.status_code for attempt in attempts] == [StatusCode.ERROR, StatusCode.OK]+ assert parent.status.status_code == StatusCode.OK+ assert any(event.name == "validation_error" for event in attempts[0].events)+ assert any(event.name == "validation_error" for event in parent.events)+ assert_usage(parent, 10, 7, 4, 6)+++def test_hidden_retry_exhaustion_redacts_all_span_failures(mode, asynchronous, exporter):+ installed = installation._current_installation()+ installation.install(+ tracer_provider=installed.tracer_provider, config=TracingConfig(hide_outputs=True, openinference_attributes=False)+ )+ responses = [ScriptedResponse(arguments='{"answer": {"secret": "private model output"}}')] * 2+ with pytest.raises(InstructorRetryException):+ run_script(mode, asynchronous, responses)+ attempts, parent = assert_attempt_spans(exporter, responses)+ assert [attempt.status.status_code for attempt in attempts] == [StatusCode.ERROR, StatusCode.ERROR]+ assert parent.status.status_code == StatusCode.ERROR+ assert "__REDACTED__" in parent.attributes["atomic_agents.llm.last_validation_error"]+ assert_usage(parent, 6, 4, 2, 4)+ for span in (*attempts, parent):+ assert "__REDACTED__" in span.status.description+ assert "private model output" not in str(span.attributes) + str(span.events) + span.status.description+ for event in span.events:+ if event.name == "exception":+ assert "__REDACTED__" in event.attributes["exception.message"]+ assert "__REDACTED__" in event.attributes["exception.stacktrace"]diff --git a/atomic-agents/tests/observability/test_llm_attempt_spans.py b/atomic-agents/tests/observability/test_llm_attempt_spans.pynew file mode 100644index 0000000..8c21b82--- /dev/null+++ b/atomic-agents/tests/observability/test_llm_attempt_spans.py@@ -0,0 +1,539 @@+import asyncio+import threading+from concurrent.futures import ThreadPoolExecutor, TimeoutError+from types import SimpleNamespace+from unittest.mock import Mock++import instructor+import pytest+from instructor.core.hooks import Hooks+from opentelemetry import context as context_api, trace as trace_api+from opentelemetry.sdk.trace import TracerProvider+from opentelemetry.sdk.trace.export import SimpleSpanProcessor+from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter+from opentelemetry.trace import SpanKind, StatusCode+from pydantic import ValidationError+from instructor.core.exceptions import InstructorRetryException++from instructor_transport import ScriptedResponse, instructor_client++from atomic_agents import AgentConfig, AtomicAgent, BaseIOSchema+from atomic_agents.observability import TracingConfig, using_conversation+from atomic_agents.observability import agent_spans, installation, llm_attempt_spans+++class Question(BaseIOSchema):+ """A test question."""++ question: str+++class Answer(BaseIOSchema):+ """A test answer."""++ answer: str+++def exporter():+ exporter = InMemorySpanExporter()+ provider = TracerProvider()+ provider.add_span_processor(SimpleSpanProcessor(exporter))+ installation.install(tracer_provider=provider, config=TracingConfig(openinference_attributes=False))+ yield exporter+ installation.uninstall()+++def usage(input_tokens=3, output_tokens=2):+ return ScriptedResponse(input_tokens=input_tokens, output_tokens=output_tokens).chat_usage()+++def finished_spans(exporter, count):+ spans = exporter.get_finished_spans()+ assert len(spans) == count+ return spans+++def response():+ return SimpleNamespace(model="response-model", usage=usage())+++def scripted_client(script):+ client = Mock(spec=instructor.Instructor)+ client.provider = instructor.Provider.OPENAI+ client.hooks = Hooks()+ client.on.side_effect = client.hooks.on+ client.off.side_effect = client.hooks.off+ client.chat = Mock()+ client.chat.completions = Mock()++ def create(**kwargs):+ script(client.hooks, kwargs)+ return Answer(answer="42")++ client.chat.completions.create.side_effect = create+ return client+++def agent(client):+ return AtomicAgent[Question, Answer](AgentConfig(client=client, model="test-model"))+++def retry_script(hooks, kwargs):+ hooks.emit_completion_arguments(**kwargs)+ hooks.emit_completion_response(response())+ hooks.emit_parse_error(ValueError("invalid output"))+ hooks.emit_completion_arguments(**kwargs)+ hooks.emit_completion_response(response())+++def run_agent(client):+ return agent(client).run(Question(question="test"))+++def test_retry_spans_usage_validation_and_parentage(exporter):+ client = instructor_client(+ instructor.Mode.TOOLS, [ScriptedResponse(arguments='{"answer": {"secret": "private output"}}'), ScriptedResponse()]+ )+ with using_conversation("conversation"):+ run_agent(client)+ first, second, parent = finished_spans(exporter, 3)+ assert [first.name, second.name] == ["chat test-model", "chat test-model"]+ assert [first.attributes["atomic_agents.llm.attempt"], second.attributes["atomic_agents.llm.attempt"]] == [1, 2]+ assert first.parent.span_id == second.parent.span_id == parent.context.span_id+ assert first.status.status_code == StatusCode.ERROR+ assert second.status.status_code == StatusCode.OK+ assert first.kind == second.kind == SpanKind.CLIENT+ assert first.events[0].name == "validation_error"+ assert first.attributes["error.type"] == "ValidationError"+ assert first.attributes["gen_ai.provider.name"] == "openai"+ assert first.attributes["gen_ai.conversation.id"] == "conversation"+ assert first.attributes["gen_ai.usage.input_tokens"] == second.attributes["gen_ai.usage.input_tokens"] == 3+ assert first.attributes["gen_ai.usage.output_tokens"] == 2+ assert first.attributes["gen_ai.usage.cache_read.input_tokens"] == 1+ assert first.attributes["gen_ai.usage.reasoning.output_tokens"] == 2+ assert parent.attributes["gen_ai.usage.input_tokens"] == 6+ assert parent.attributes["gen_ai.usage.output_tokens"] == 4+ assert parent.attributes.get("atomic_agents.llm.attempts") == 2+ assert "private output" in parent.attributes["atomic_agents.llm.last_validation_error"]+ assert parent.attributes["gen_ai.response.model"] == first.attributes["gen_ai.response.model"] == "response-model"+ event = parent.events[0]+ assert event.name == "validation_error"+ assert event.attributes["atomic_agents.llm.attempt"] == 1+++def test_disabled_attempt_spans_keep_totals(exporter):+ installed = installation._current_installation()+ installation.install(tracer_provider=installed.tracer_provider, config=TracingConfig(llm_spans=False))+ run_agent(scripted_client(retry_script))+ (parent,) = finished_spans(exporter, 1)+ assert parent.attributes.get("atomic_agents.llm.attempts") == 2+ assert parent.attributes["gen_ai.usage.input_tokens"] == 6+++def test_completion_error_is_bounded_and_ends_attempt(exporter):+ installed = installation._current_installation()+ installation.install(tracer_provider=installed.tracer_provider, config=TracingConfig(max_attribute_length=32))+ error = ValueError("failed " * 20)+ expected = str(error)[:18] + "...[truncated]"++ def script(hooks, kwargs):+ hooks.emit_completion_arguments(**kwargs)+ hooks.emit_parse_error(error)+ hooks.emit_completion_error(error, attempt_number=1, max_attempts=1, is_last_attempt=True)++ run_agent(scripted_client(script))+ child, parent = finished_spans(exporter, 2)+ assert child.status.status_code == StatusCode.ERROR+ assert child.attributes["error.type"] == "ValueError"+ assert child.events[-1].attributes["exception.message"] == expected+ assert len(child.events[-1].attributes["exception.stacktrace"]) <= 32+ assert child.status.description == expected+ assert len(child.status.description) == 32+ assert parent.attributes["atomic_agents.llm.last_validation_error"] == expected+ assert parent.attributes["atomic_agents.llm.last_validation_error.original_length"] == len(str(error))+ assert parent.status.status_code == StatusCode.OK+++def test_registration_once_and_uninstall_preserves_application_hooks(exporter):+ client = scripted_client(retry_script)+ application_hook = Mock()+ client.hooks.on("completion:response", application_hook)+ for _ in range(3):+ run_agent(client)+ assert client.on.call_count == 4+ assert application_hook.call_count == 6+ installation.uninstall()+ assert client.off.call_count == 4+ client.hooks.emit_completion_response(response())+ assert application_hook.call_count == 7+ assert not llm_attempt_spans._clients++[email protected]("asynchronous", [False, True])+def test_concurrent_shared_client_runs_are_isolated(exporter, asynchronous):+ client = scripted_client(retry_script)++ async def invoke(number):+ async def complete(agent, *args, **kwargs):+ client.hooks.emit_completion_arguments(model="test-model")+ await asyncio.sleep(0)+ client.hooks.emit_completion_response(SimpleNamespace(model="response-model", usage=usage(number)))+ return Answer(answer="ok")++ if asynchronous:+ return await agent_spans._trace_agent_async_call(complete)(agent(client))+ return await asyncio.to_thread(run_agent, client)++ async def gather():+ await asyncio.gather(invoke(10), invoke(20))++ asyncio.run(gather())+ parents = [span for span in exporter.get_finished_spans() if span.name.startswith("invoke_agent")]+ expected = [10, 20] if asynchronous else [6, 6]+ assert sorted(span.attributes.get("gen_ai.usage.input_tokens", -1) for span in parents) == expected+ assert all(span.attributes["atomic_agents.llm.attempts"] == (1 if asynchronous else 2) for span in parents)++[email protected]("kind", ["sync", "async", "stream", "async_stream"])[email protected]("fails", [False, True])+def test_open_attempt_ends_at_agent_end(exporter, kind, fails):+ client = scripted_client(retry_script)++ def start():+ client.hooks.emit_completion_arguments(model="test-model")+ if fails:+ raise ValueError("agent failed")+ return Answer(answer="partial")++ def call(*args, **kwargs):+ return start()++ async def async_call(*args, **kwargs):+ return start()++ def stream(*args, **kwargs):+ yield start()+ yield Answer(answer="final")++ async def async_stream(*args, **kwargs):+ yield start()+ yield Answer(answer="final")++ def invoke():+ if kind == "sync":+ return agent_spans._trace_agent_call(call)(agent(client))+ if kind == "async":+ return asyncio.run(agent_spans._trace_agent_async_call(async_call)(agent(client)))+ if kind == "stream":+ iterator = agent_spans._trace_agent_stream(stream)(agent(client))+ next(iterator)+ assert trace_api.get_current_span() is trace_api.INVALID_SPAN+ iterator.close()+ return None+ return asyncio.run(consume_async_stream())++ async def consume_async_stream():+ iterator = agent_spans._trace_agent_async_stream(async_stream)(agent(client))+ await anext(iterator)+ assert trace_api.get_current_span() is trace_api.INVALID_SPAN+ await iterator.aclose()++ if fails:+ with pytest.raises(ValueError, match="agent failed"):+ invoke()+ else:+ invoke()+ child, parent = finished_spans(exporter, 2)+ assert child.status.status_code == StatusCode.UNSET+ assert child.parent.span_id == parent.context.span_id+ assert child.end_time <= parent.end_time+++def test_validation_text_redacted_and_anthropic_usage(exporter):+ installed = installation._current_installation()+ installation.install(tracer_provider=installed.tracer_provider, config=TracingConfig(hide_outputs=True))++ def script(hooks, kwargs):+ hooks.emit_completion_arguments(**kwargs)+ with pytest.raises(ValidationError) as invalid:+ Answer.model_validate({"answer": {"secret": "private output"}})+ hooks.emit_completion_response(SimpleNamespace(model="response-model", usage=usage(9, 4)))+ hooks.emit_parse_error(invalid.value)+ hooks.emit_completion_error(invalid.value, attempt_number=1, max_attempts=1, is_last_attempt=True)++ run_agent(scripted_client(script))+ child, parent = finished_spans(exporter, 2)+ assert parent.attributes["atomic_agents.llm.last_validation_error"] == "__REDACTED__"+ assert parent.events[0].attributes["atomic_agents.llm.last_validation_error"] == "__REDACTED__"+ assert parent.attributes["gen_ai.usage.input_tokens"] == 9+ assert child.status.description == "ValidationError: 1 errors (__REDACTED__)"+ assert child.events[-1].attributes["exception.message"] == child.status.description+ assert "ValidationError: 1 errors (__REDACTED__)" in child.events[-1].attributes["exception.stacktrace"]+ assert "private output" not in str(child.events)+++def test_hook_failure_warns_without_breaking_agent(exporter, caplog):+ class BrokenResponse:+ @property+ def model(self):+ raise RuntimeError("broken metadata")++ def script(hooks, kwargs):+ hooks.emit_completion_arguments(**kwargs)+ hooks.emit_completion_response(BrokenResponse())++ assert run_agent(scripted_client(script)).answer == "42"+ child, parent = finished_spans(exporter, 2)+ assert child.status.status_code == StatusCode.UNSET+ assert "Instructor tracing hook failed" in caplog.text++[email protected]("asynchronous", [False, True])+def test_stream_attempt_context_survives_cursor_advances(exporter, asynchronous):+ client = scripted_client(retry_script)++ def begin():+ client.hooks.emit_completion_arguments(model="test-model")+ return trace_api.get_current_span().get_span_context().span_id++ def complete(expected):+ assert trace_api.get_current_span().get_span_context().span_id == expected+ client.hooks.emit_completion_response(response())+ return Answer(answer="final")++ def stream(*args, **kwargs):+ expected = begin()+ yield Answer(answer="partial")+ yield complete(expected)++ async def async_stream(*args, **kwargs):+ expected = begin()+ yield Answer(answer="partial")+ await asyncio.sleep(0)+ yield complete(expected)++ async def consume():+ return [partial async for partial in agent_spans._trace_agent_async_stream(async_stream)(agent(client))]++ if asynchronous:+ assert len(asyncio.run(consume())) == 2+ else:+ assert len(list(agent_spans._trace_agent_stream(stream)(agent(client)))) == 2+ child, parent = finished_spans(exporter, 2)+ assert child.status.status_code == StatusCode.OK+ assert parent.status.status_code == StatusCode.OK+++def test_validation_text_truncation_and_extra_hook_keywords(exporter):+ installed = installation._current_installation()+ installation.install(tracer_provider=installed.tracer_provider, config=TracingConfig(max_attribute_length=4))++ def script(hooks, kwargs):+ hooks.emit_completion_arguments(**kwargs)+ llm_attempt_spans._parse_error(ValueError("long error"), future_option=True)+ llm_attempt_spans._completion_response(response(), future_option=True)++ run_agent(scripted_client(script))+ child, parent = finished_spans(exporter, 2)+ assert parent.attributes["atomic_agents.llm.last_validation_error"] == "...["+ assert parent.attributes["atomic_agents.llm.last_validation_error.original_length"] == 10+++def test_threaded_first_use_registers_once_and_uninstalls(exporter, monkeypatch):+ client = scripted_client(retry_script)+ first_registration = threading.Barrier(2, timeout=5)+ release_registration = threading.Event()+ original_on = Hooks.on++ def blocked_on(hooks, event, handler):+ if event == "completion:kwargs":+ first_registration.wait()+ assert release_registration.wait(timeout=5)+ return original_on(hooks, event, handler)++ monkeypatch.setattr(Hooks, "on", blocked_on)+ client.on.side_effect = client.hooks.on+ with ThreadPoolExecutor(max_workers=2) as executor:+ first = executor.submit(run_agent, client)+ first_registration.wait()+ second_started = threading.Event()++ def second_run():+ second_started.set()+ return run_agent(client)++ second = executor.submit(second_run)+ assert second_started.wait(timeout=5)+ try:+ with pytest.raises(TimeoutError):+ second.result(timeout=0.1)+ assert client.on.call_count == 1+ finally:+ release_registration.set()+ first.result(timeout=5)+ second.result(timeout=5)+ assert len(client.hooks._handlers) == 4+ assert all(len(handlers) == 1 for handlers in client.hooks._handlers.values())+ installation.uninstall()+ assert all(not handlers for handlers in client.hooks._handlers.values())+++def test_partial_registration_is_rolled_back(exporter):+ client = scripted_client(retry_script)++ def fail_registration(event, handler):+ if event == "parse:error":+ raise RuntimeError("registration failed")+ client.hooks.on(event, handler)++ client.on.side_effect = fail_registration+ llm_attempt_spans.register_client(client)+ assert client not in llm_attempt_spans._clients+ assert all(not handlers for handlers in client.hooks._handlers.values())+ client.on.side_effect = client.hooks.on+ llm_attempt_spans.register_client(client)+ assert all(len(handlers) == 1 for handlers in client.hooks._handlers.values())++[email protected]("completion", ["response", "error"])+def test_suppressed_attempt_events_are_ignored_then_normal_attempt_works(exporter, completion):+ def script(hooks, kwargs):+ run = llm_attempt_spans._active_run.get()+ token = context_api.attach(context_api.set_value("suppress_instrumentation", True))+ try:+ hooks.emit_completion_arguments(**kwargs)+ finally:+ context_api.detach(token)+ if completion == "response":+ hooks.emit_completion_response(response())+ else:+ hooks.emit_completion_error(ValueError("suppressed"), attempt_number=1, max_attempts=2, is_last_attempt=False)+ hooks.emit_parse_error(ValueError("suppressed"))+ assert run.attempt_count == 0+ assert run.open_attempt is None+ assert not exporter.get_finished_spans()+ hooks.emit_completion_arguments(**kwargs)+ hooks.emit_completion_response(response())++ run_agent(scripted_client(script))+ child, parent = finished_spans(exporter, 2)+ assert child.attributes["atomic_agents.llm.attempt"] == 1+ assert parent.attributes["atomic_agents.llm.attempts"] == 1+ assert parent.attributes["gen_ai.usage.input_tokens"] == 3+ assert not parent.events+++ "mode", [instructor.Mode.TOOLS, instructor.Mode.JSON, instructor.Mode.JSON_SCHEMA, instructor.Mode.RESPONSES_TOOLS]+)+def test_real_transport_usage_is_accumulated_per_response(exporter, mode):+ client = instructor_client(+ mode,+ [+ ScriptedResponse(arguments='{"answer": {"private": "model text"}}'),+ ScriptedResponse(input_tokens=7, output_tokens=5, cached_tokens=3, reasoning_tokens=4),+ ],+ )+ assert run_agent(client).answer == "42"+ first, second, parent = finished_spans(exporter, 3)+ assert first.status.status_code == StatusCode.ERROR+ assert first.events[0].name == "validation_error"+ assert second.status.status_code == StatusCode.OK+ assert first.attributes["gen_ai.usage.input_tokens"] == 3+ assert second.attributes["gen_ai.usage.input_tokens"] == 7+ assert parent.attributes["gen_ai.usage.input_tokens"] == 10+ assert parent.attributes["gen_ai.usage.output_tokens"] == 7+ assert parent.attributes["gen_ai.usage.cache_read.input_tokens"] == 4+ assert parent.attributes["gen_ai.usage.reasoning.output_tokens"] == 6+ assert parent.attributes["atomic_agents.llm.attempts"] == 2+ assert "completion:usage" not in llm_attempt_spans._HANDLERS++[email protected]("hidden", [False, True])+def test_real_retry_exhaustion_redacts_or_bounds_every_exception(exporter, hidden):+ installed = installation._current_installation()+ installation.install(+ tracer_provider=installed.tracer_provider, config=TracingConfig(hide_outputs=hidden, max_attribute_length=256)+ )+ secret = "private model text " * 100+ client = instructor_client(+ instructor.Mode.TOOLS, [ScriptedResponse(arguments='{"answer": {"secret": "' + secret + '"}}')] * 3+ )+ configured = agent(client)+ configured.model_api_parameters["max_retries"] = 2+ with pytest.raises(InstructorRetryException):+ configured.run(Question(question="test"))+ first, second, third, parent = finished_spans(exporter, 4)+ assert parent.status.status_code == StatusCode.ERROR+ assert parent.attributes["atomic_agents.llm.attempts"] == 3+ assert parent.attributes["gen_ai.usage.input_tokens"] == 9+ message = parent.events[-1].attributes["exception.message"]+ assert len(message) <= 256+ assert len(parent.status.description) <= 256+ assert len(parent.events[-1].attributes["exception.stacktrace"]) <= 256+ if hidden:+ assert message == parent.status.description == "InstructorRetryException: 3 attempts (__REDACTED__)"+ assert "private model text" not in str(parent.events)+ assert "private model text" not in str(first.events) + str(second.events) + str(third.events)+ assert (+ first.events[0].attributes["atomic_agents.llm.last_validation_error"] == "ValidationError: 1 errors (__REDACTED__)"+ )+ else:+ assert message.endswith("...[truncated]")+++def test_direct_instructor_calls_inside_run_add_to_agent_totals(exporter):+ client = instructor_client(instructor.Mode.TOOLS, [ScriptedResponse(), ScriptedResponse(input_tokens=11)])++ def call_twice(configured):+ for _ in range(2):+ client.chat.completions.create(+ model="test-model", response_model=Answer, messages=[{"role": "user", "content": "test"}]+ )+ return Answer(answer="42")++ agent_spans._trace_agent_call(call_twice)(agent(client))+ first, second, parent = finished_spans(exporter, 3)+ assert parent.attributes["gen_ai.usage.input_tokens"] == 14+ assert parent.attributes["atomic_agents.llm.attempts"] == 2+ assert first.parent.span_id == second.parent.span_id == parent.context.span_id+++def test_missing_model_and_final_completion_error_still_count(exporter):+ def script(hooks, kwargs):+ hooks.emit_completion_arguments()+ hooks.emit_completion_response(response())+ hooks.emit_completion_arguments()+ hooks.emit_completion_error(RuntimeError("transport failed"))++ run_agent(scripted_client(script))+ first, second, parent = finished_spans(exporter, 3)+ assert first.name == second.name == "chat"+ assert first.kind == second.kind == SpanKind.CLIENT+ assert second.status.status_code == StatusCode.ERROR+ assert parent.attributes["atomic_agents.llm.attempts"] == 2+ assert parent.attributes["gen_ai.usage.input_tokens"] == 3++[email protected]("chain", ["__cause__", "__context__"])+def test_wrapped_validation_error_and_tool_exception_are_redacted(exporter, chain):+ from atomic_agents.observability import tool_spans+ from atomic_agents.base.base_tool import BaseTool++ installed = installation._current_installation()+ installation.install(tracer_provider=installed.tracer_provider, config=TracingConfig(hide_outputs=True))+ with pytest.raises(ValidationError) as invalid:+ Answer.model_validate({"answer": {"secret": "private model text"}})+ wrapped = RuntimeError("private wrapper text")+ setattr(wrapped, chain, invalid.value)+ with pytest.raises(RuntimeError), tool_spans._tool_span(Mock(spec=BaseTool, tool_name="test"), (), {}):+ raise wrapped+ (span,) = finished_spans(exporter, 1)+ assert span.status.description == "RuntimeError: 1 errors (__REDACTED__)"+ assert span.events[0].attributes["exception.message"] == span.status.description+ stacktrace = span.events[0].attributes["exception.stacktrace"]+ assert "Traceback (most recent call last)" in stacktrace+ assert stacktrace.rstrip().endswith(span.status.description)+ assert "private model text" not in stacktrace+ assert "private wrapper text" not in stacktracediff --git a/atomic-agents/tests/observability/test_openinference_tracing.py b/atomic-agents/tests/observability/test_openinference_tracing.pydeleted file mode 100644index 6fb3bc8..0000000--- a/atomic-agents/tests/observability/test_openinference_tracing.py+++ /dev/null@@ -1,851 +0,0 @@-import abc-import asyncio-import contextvars-import gc-import json-import logging-import types-from concurrent.futures import ThreadPoolExecutor-from importlib.metadata import entry_points-from typing import Dict, List-from unittest.mock import Mock--import instructor-import pytest-from openinference.instrumentation import OITracer, TraceConfig, using_metadata, using_session-from openinference.semconv.trace import OpenInferenceSpanKindValues, SpanAttributes-from opentelemetry import trace as trace_api-from opentelemetry.sdk.trace import ReadableSpan, TracerProvider-from opentelemetry.sdk.trace.export import SimpleSpanProcessor-from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter-from opentelemetry.trace import StatusCode-from pydantic import Field--from atomic_agents import AgentConfig, AtomicAgent, BaseIOSchema, BaseTool-from atomic_agents.context import BaseDynamicContextProvider, SystemPromptGenerator-from atomic_agents.observability import AtomicAgentsInstrumentor-import atomic_agents.observability as observability-from atomic_agents.observability import openinference_tracing as tracing---class QuestionSchema(BaseIOSchema):- """A question for the agent."""-- question: str = Field(...)---class AnswerSchema(BaseIOSchema):- """An answer from the agent."""-- answer: str = Field(...)---class CalculationInputSchema(BaseIOSchema):- """Expression to evaluate."""-- expression: str = Field(...)---class CalculationOutputSchema(BaseIOSchema):- """Result of the evaluation."""-- result: str = Field(...)---class CalculatorTool(BaseTool[CalculationInputSchema, CalculationOutputSchema]):- def run(self, params: CalculationInputSchema, precision: int = 2) -> CalculationOutputSchema:- return CalculationOutputSchema(result=f"{len(params.expression):.{precision}f}")---class AsyncCalculatorTool(BaseTool[CalculationInputSchema, CalculationOutputSchema]):- def run(self, params: CalculationInputSchema) -> CalculationOutputSchema:- return asyncio.run(self.arun(params))-- async def arun(self, params: CalculationInputSchema) -> CalculationOutputSchema:- return CalculationOutputSchema(result=str(len(params.expression)))---class ThreadedCalculatorTool(BaseTool[CalculationInputSchema, CalculationOutputSchema]):- def run(self, params: CalculationInputSchema) -> CalculationOutputSchema:- return CalculationOutputSchema(result=str(len(params.expression)))-- async def run_async(self, params: CalculationInputSchema) -> CalculationOutputSchema:- return await asyncio.to_thread(self.run, params)---class RunMixin:- def run(self, params: CalculationInputSchema) -> CalculationOutputSchema:- return CalculationOutputSchema(result="mixin")---class MixinCalculatorTool(RunMixin, BaseTool[CalculationInputSchema, CalculationOutputSchema]):- pass---class StaticRunTool(BaseTool[CalculationInputSchema, CalculationOutputSchema]):- @staticmethod- def run(params: CalculationInputSchema) -> CalculationOutputSchema:- return CalculationOutputSchema(result="static")---class FailingTool(BaseTool[CalculationInputSchema, CalculationOutputSchema]):- def run(self, params: CalculationInputSchema) -> CalculationOutputSchema:- raise ValueError("boom")---class AnswerStore:- """Container object pydantic cannot serialize to JSON."""-- def __init__(self):- self.values: List[int] = []---class UnserializableInputSchema(BaseIOSchema):- """Input carrying an arbitrary object."""-- model_config = {"arbitrary_types_allowed": True}- store: AnswerStore = Field(...)---class EchoTool(BaseTool[UnserializableInputSchema, CalculationOutputSchema]):- def run(self, params: UnserializableInputSchema) -> CalculationOutputSchema:- return CalculationOutputSchema(result="echo")---def exporter():- exporter = InMemorySpanExporter()- provider = TracerProvider()- provider.add_span_processor(SimpleSpanProcessor(exporter))- exporter.provider = provider- instrumentor = AtomicAgentsInstrumentor()- instrumentor.instrument(tracer_provider=provider)- yield exporter- instrumentor.uninstrument()---def provider_tracer(exporter):- return exporter.provider.get_tracer("test")---def sync_client(response=None, partials=None):- client = Mock(spec=instructor.Instructor)- client.chat = Mock()- client.chat.completions = Mock()- client.chat.completions.create = Mock(return_value=response or AnswerSchema(answer="42"))-- def create_partial(**kwargs):- for partial in partials or [AnswerSchema(answer="4"), AnswerSchema(answer="42")]:- yield partial-- client.chat.completions.create_partial = create_partial- return client---def async_client(response=None, partials=None):- client = Mock(spec=instructor.AsyncInstructor)- client.chat = Mock()- client.chat.completions = Mock()-- async def create(**kwargs):- return response or AnswerSchema(answer="42")-- async def create_partial(**kwargs):- for partial in partials or [AnswerSchema(answer="4"), AnswerSchema(answer="42")]:- yield partial-- client.chat.completions.create = create- client.chat.completions.create_partial = create_partial- return client---def make_agent(client) -> AtomicAgent[QuestionSchema, AnswerSchema]:- return AtomicAgent[QuestionSchema, AnswerSchema](AgentConfig(client=client, model="test-model"))---def spans_of(exporter) -> List[ReadableSpan]:- return list(exporter.get_finished_spans())---def kind_of(span: ReadableSpan) -> str:- return span.attributes[SpanAttributes.OPENINFERENCE_SPAN_KIND]---def input_json(span: ReadableSpan) -> Dict:- return json.loads(span.attributes[SpanAttributes.INPUT_VALUE])---def output_json(span: ReadableSpan) -> Dict:- return json.loads(span.attributes[SpanAttributes.OUTPUT_VALUE])---def test_run_emits_agent_span_with_io_and_framework_attributes(exporter):- agent = make_agent(sync_client())-- response = agent.run(QuestionSchema(question="6*7?"))-- assert response.answer == "42"- (span,) = spans_of(exporter)- assert span.name == "AtomicAgent[QuestionSchema, AnswerSchema].run"- assert kind_of(span) == OpenInferenceSpanKindValues.AGENT.value- assert span.attributes[SpanAttributes.AGENT_NAME] == "AtomicAgent[QuestionSchema, AnswerSchema]"- assert input_json(span) == {"question": "6*7?"}- assert output_json(span) == {"answer": "42"}- assert span.attributes[SpanAttributes.INPUT_MIME_TYPE] == "application/json"- assert span.attributes[SpanAttributes.OUTPUT_MIME_TYPE] == "application/json"- assert span.attributes[tracing.ATTR_MODEL] == "test-model"- assert span.attributes[tracing.ATTR_MODE] == agent.mode.value- assert span.attributes[tracing.ATTR_INPUT_SCHEMA] == "QuestionSchema"- assert span.attributes[tracing.ATTR_OUTPUT_SCHEMA] == "AnswerSchema"- assert span.attributes[tracing.ATTR_HISTORY_MESSAGE_COUNT] == 0- assert span.attributes[tracing.ATTR_CONTEXT_PROVIDERS] == ()- assert len(span.attributes[tracing.ATTR_SYSTEM_PROMPT_SHA256]) == 64- assert span.status.status_code == StatusCode.OK---def test_second_turn_reports_history_size_entering_the_turn(exporter):- agent = make_agent(sync_client())- agent.run(QuestionSchema(question="first"))-- agent.run(QuestionSchema(question="second"))-- first, second = spans_of(exporter)- assert first.attributes[tracing.ATTR_HISTORY_MESSAGE_COUNT] == 0- assert second.attributes[tracing.ATTR_HISTORY_MESSAGE_COUNT] == 2---def test_agent_without_system_role_has_no_prompt_hash(exporter):- agent = AtomicAgent[QuestionSchema, AnswerSchema](AgentConfig(client=sync_client(), model="m", system_role=None))-- agent.run(QuestionSchema(question="?"))-- (span,) = spans_of(exporter)- assert tracing.ATTR_SYSTEM_PROMPT_SHA256 not in span.attributes---def test_abstract_intermediate_tool_base_is_not_wrapped(exporter):- class AbstractCalculator(BaseTool[CalculationInputSchema, CalculationOutputSchema]):- @abc.abstractmethod- def run(self, params: CalculationInputSchema) -> CalculationOutputSchema:- raise NotImplementedError-- assert not hasattr(AbstractCalculator.__dict__["run"], tracing.TRACED_MARKER)---def test_run_without_input_sets_no_input_attribute(exporter):- agent = make_agent(sync_client())-- agent.run()-- (span,) = spans_of(exporter)- assert SpanAttributes.INPUT_VALUE not in span.attributes- assert output_json(span) == {"answer": "42"}---def test_run_failure_propagates_and_marks_span_error(exporter):- client = sync_client()- client.chat.completions.create = Mock(side_effect=RuntimeError("provider down"))- agent = make_agent(client)-- with pytest.raises(RuntimeError, match="provider down"):- agent.run(QuestionSchema(question="?"))-- (span,) = spans_of(exporter)- assert span.status.status_code == StatusCode.ERROR- assert [event.name for event in span.events] == ["exception"]---def test_subclassed_agent_is_named_after_the_subclass_and_emits_one_span(exporter):- class ResearchAgent(AtomicAgent[QuestionSchema, AnswerSchema]):- def run(self, user_input=None):- return super().run(user_input)-- agent = ResearchAgent(AgentConfig(client=sync_client(), model="test-model"))-- agent.run(QuestionSchema(question="?"))-- (span,) = spans_of(exporter)- assert span.name == "ResearchAgent[QuestionSchema, AnswerSchema].run"---def test_agent_span_nests_under_a_user_chain_span(exporter, provider_tracer):- agent = make_agent(sync_client())-- with provider_tracer.start_as_current_span("turn") as turn:- agent.run(QuestionSchema(question="?"))-- agent_span, turn_span = spans_of(exporter)- assert agent_span.parent.span_id == turn.get_span_context().span_id- assert turn_span.name == "turn"---def test_manual_setup_from_the_guide_groups_agents_and_tools_under_one_chain_span():- exporter = InMemorySpanExporter()- tracer_provider = TracerProvider()- tracer_provider.add_span_processor(SimpleSpanProcessor(exporter))- trace_config = TraceConfig()- instrumentor = AtomicAgentsInstrumentor()- instrumentor.instrument(tracer_provider=tracer_provider, config=trace_config)- tracer = OITracer(tracer_provider.get_tracer(__name__), config=trace_config)- planner = make_agent(sync_client())- answerer = make_agent(sync_client())- calculator = CalculatorTool()- try:- with tracer.start_as_current_span("answer_question", openinference_span_kind="chain") as turn:- turn.set_input("what is 2 + 2?")- plan = planner.run(QuestionSchema(question="what is 2 + 2?"))- calculation = calculator.run(CalculationInputSchema(expression=plan.answer))- answer = answerer.run(QuestionSchema(question=str(calculation.result)))- turn.set_output(answer.answer)- finally:- instrumentor.uninstrument()-- spans = spans_of(exporter)- chain_span = next(span for span in spans if kind_of(span) == OpenInferenceSpanKindValues.CHAIN.value)- children = [span for span in spans if span is not chain_span]- assert [kind_of(span) for span in children] == ["AGENT", "TOOL", "AGENT"]- assert all(span.parent.span_id == chain_span.get_span_context().span_id for span in children)- assert chain_span.attributes[SpanAttributes.INPUT_VALUE] == "what is 2 + 2?"- assert chain_span.attributes[SpanAttributes.OUTPUT_VALUE] == answer.answer---def test_llm_call_inside_run_becomes_child_of_agent_span(exporter, provider_tracer):- def create(**kwargs):- with provider_tracer.start_as_current_span("llm"):- return AnswerSchema(answer="42")-- client = sync_client()- client.chat.completions.create = create- agent = make_agent(client)-- agent.run(QuestionSchema(question="?"))-- llm_span, agent_span = spans_of(exporter)- assert llm_span.parent.span_id == agent_span.context.span_id---def test_run_stream_records_final_output_and_keeps_child_context_per_step(exporter, provider_tracer):- current_between_yields = []-- def create_partial(**kwargs):- yield AnswerSchema(answer="4")- with provider_tracer.start_as_current_span("llm-chunk"):- yield AnswerSchema(answer="42")-- client = sync_client()- client.chat.completions.create_partial = create_partial- agent = make_agent(client)-- partials = []- for partial in agent.run_stream(QuestionSchema(question="?")):- partials.append(partial.answer)- current_between_yields.append(trace_api.get_current_span())-- assert partials == ["4", "42"]- assert all(span is trace_api.INVALID_SPAN for span in current_between_yields)- chunk_span, agent_span = spans_of(exporter)- assert agent_span.name == "AtomicAgent[QuestionSchema, AnswerSchema].run_stream"- assert chunk_span.parent.span_id == agent_span.context.span_id- assert output_json(agent_span) == {"answer": "42"}- assert agent_span.status.status_code == StatusCode.OK---def test_run_stream_preserves_the_generator_return_value(exporter):- agent = make_agent(sync_client())-- def consume():- final = yield from agent.run_stream(QuestionSchema(question="?"))- return final-- generator = consume()- with pytest.raises(StopIteration) as stop:- while True:- next(generator)-- assert stop.value.value.answer == "42"---def test_run_stream_with_no_partials_ends_ok_without_output(exporter):- client = sync_client()- client.chat.completions.create_partial = lambda **kwargs: iter([])- agent = make_agent(client)-- assert list(agent.run_stream(QuestionSchema(question="?"))) == []-- (span,) = spans_of(exporter)- assert span.status.status_code == StatusCode.OK- assert SpanAttributes.OUTPUT_VALUE not in span.attributes---def test_run_stream_closed_early_ends_span_and_closes_inner_generator(exporter):- inner_closed = []-- def create_partial(**kwargs):- try:- yield AnswerSchema(answer="4")- yield AnswerSchema(answer="42")- finally:- inner_closed.append(True)-- client = sync_client()- client.chat.completions.create_partial = create_partial- agent = make_agent(client)-- for partial in agent.run_stream(QuestionSchema(question="?")):- break-- assert inner_closed == [True]- (span,) = spans_of(exporter)- assert span.attributes[tracing.ATTR_STREAM_CLOSED_EARLY] is True- assert output_json(span) == {"answer": "4"}- assert span.status.status_code == StatusCode.OK---def test_run_stream_failure_mid_stream_marks_error(exporter):- def create_partial(**kwargs):- yield AnswerSchema(answer="4")- raise RuntimeError("stream broke")-- client = sync_client()- client.chat.completions.create_partial = create_partial- agent = make_agent(client)-- with pytest.raises(RuntimeError, match="stream broke"):- list(agent.run_stream(QuestionSchema(question="?")))-- (span,) = spans_of(exporter)- assert span.status.status_code == StatusCode.ERROR---async def test_run_async_emits_agent_span(exporter):- agent = make_agent(async_client())-- response = await agent.run_async(QuestionSchema(question="?"))-- assert response.answer == "42"- (span,) = spans_of(exporter)- assert span.name == "AtomicAgent[QuestionSchema, AnswerSchema].run_async"- assert output_json(span) == {"answer": "42"}- assert span.status.status_code == StatusCode.OK---async def test_run_async_stream_records_last_partial_and_nests_children(exporter, provider_tracer):- async def create_partial(**kwargs):- yield AnswerSchema(answer="4")- with provider_tracer.start_as_current_span("llm-chunk"):- yield AnswerSchema(answer="42")-- client = async_client()- client.chat.completions.create_partial = create_partial- agent = make_agent(client)-- partials = [partial.answer async for partial in agent.run_async_stream(QuestionSchema(question="?"))]-- assert partials == ["4", "42"]- chunk_span, agent_span = spans_of(exporter)- assert agent_span.name == "AtomicAgent[QuestionSchema, AnswerSchema].run_async_stream"- assert chunk_span.parent.span_id == agent_span.context.span_id- assert output_json(agent_span) == {"answer": "42"}---async def test_run_async_stream_closed_early_ends_span(exporter):- agent = make_agent(async_client())-- stream = agent.run_async_stream(QuestionSchema(question="?"))- first = await stream.__anext__()- await stream.aclose()-- assert first.answer == "4"- (span,) = spans_of(exporter)- assert span.attributes[tracing.ATTR_STREAM_CLOSED_EARLY] is True- assert span.status.status_code == StatusCode.OK---async def test_concurrent_async_runs_keep_separate_spans(exporter):- first = make_agent(async_client(response=AnswerSchema(answer="one")))- second = make_agent(async_client(response=AnswerSchema(answer="two")))-- await asyncio.gather(first.run_async(QuestionSchema(question="1")), second.run_async(QuestionSchema(question="2")))-- outputs = {input_json(span)["question"]: output_json(span)["answer"] for span in spans_of(exporter)}- assert outputs == {"1": "one", "2": "two"}- assert all(span.parent is None for span in spans_of(exporter))---def test_tool_defined_before_instrumentation_emits_tool_span(exporter):- tool = CalculatorTool()-- result = tool.run(CalculationInputSchema(expression="6*7"), precision=1)-- assert result.result == "3.0"- (span,) = spans_of(exporter)- assert span.name == "CalculatorTool"- assert kind_of(span) == OpenInferenceSpanKindValues.TOOL.value- assert span.attributes[SpanAttributes.TOOL_NAME] == "CalculationInputSchema"- assert span.attributes[SpanAttributes.TOOL_DESCRIPTION] == "Expression to evaluate."- assert json.loads(span.attributes[SpanAttributes.TOOL_PARAMETERS])["title"] == "CalculationInputSchema"- assert input_json(span) == {"expression": "6*7"}- assert output_json(span) == {"result": "3.0"}- assert span.status.status_code == StatusCode.OK---def test_tool_accepts_keyword_params(exporter):- result = CalculatorTool().run(params=CalculationInputSchema(expression="1+1"))-- assert result.result == "3.00"- (span,) = spans_of(exporter)- assert input_json(span) == {"expression": "1+1"}---def test_tool_defined_after_instrumentation_emits_tool_span(exporter):- class LateTool(BaseTool[CalculationInputSchema, CalculationOutputSchema]):- def run(self, params: CalculationInputSchema) -> CalculationOutputSchema:- return CalculationOutputSchema(result="late")-- LateTool().run(CalculationInputSchema(expression="x"))-- (span,) = spans_of(exporter)- assert span.name == "LateTool"- assert LateTool().input_schema is CalculationInputSchema---def test_subclass_calling_super_run_emits_one_span(exporter):- class LoudCalculatorTool(CalculatorTool):- def run(self, params: CalculationInputSchema, precision: int = 2) -> CalculationOutputSchema:- return super().run(params, precision)-- LoudCalculatorTool().run(CalculationInputSchema(expression="2*2"))-- (span,) = spans_of(exporter)- assert span.name == "LoudCalculatorTool"---def test_sync_run_delegating_to_arun_emits_one_span(exporter):- AsyncCalculatorTool().run(CalculationInputSchema(expression="2*2"))-- (span,) = spans_of(exporter)- assert span.name == "AsyncCalculatorTool"- assert output_json(span) == {"result": "3"}---async def test_run_async_delegating_to_thread_emits_one_span(exporter):- await ThreadedCalculatorTool().run_async(CalculationInputSchema(expression="3*3"))-- (span,) = spans_of(exporter)- assert span.name == "ThreadedCalculatorTool"- assert output_json(span) == {"result": "3"}---def test_mixin_defined_run_is_traced(exporter):- MixinCalculatorTool().run(CalculationInputSchema(expression="x"))-- (span,) = spans_of(exporter)- assert span.name == "MixinCalculatorTool"---def test_static_run_is_left_alone(exporter):- assert StaticRunTool.run(CalculationInputSchema(expression="x")).result == "static"- assert spans_of(exporter) == []---def test_tool_failure_propagates_same_exception_and_marks_error(exporter):- with pytest.raises(ValueError, match="boom") as raised:- FailingTool().run(CalculationInputSchema(expression="x"))-- (span,) = spans_of(exporter)- assert span.status.status_code == StatusCode.ERROR- assert raised.value.args == ("boom",)---def test_unserializable_input_still_returns_result(exporter, caplog):- with caplog.at_level(logging.WARNING):- result = EchoTool().run(UnserializableInputSchema(store=AnswerStore()))-- assert result.result == "echo"- (span,) = spans_of(exporter)- assert span.status.status_code == StatusCode.OK---def test_agent_run_inside_tool_nests_under_tool_span(exporter):- agent = make_agent(sync_client())-- class AgentTool(BaseTool[CalculationInputSchema, CalculationOutputSchema]):- def run(self, params: CalculationInputSchema) -> CalculationOutputSchema:- return CalculationOutputSchema(result=agent.run(QuestionSchema(question=params.expression)).answer)-- AgentTool().run(CalculationInputSchema(expression="?"))-- agent_span, tool_span = spans_of(exporter)- assert agent_span.parent.span_id == tool_span.context.span_id---def test_hide_inputs_and_outputs_redacts_values_and_drops_mime_types():- exporter = InMemorySpanExporter()- provider = TracerProvider()- provider.add_span_processor(SimpleSpanProcessor(exporter))- instrumentor = AtomicAgentsInstrumentor()- instrumentor.instrument(tracer_provider=provider, config=TraceConfig(hide_inputs=True, hide_outputs=True))- try:- make_agent(sync_client()).run(QuestionSchema(question="secret"))- CalculatorTool().run(CalculationInputSchema(expression="1+1"))- finally:- instrumentor.uninstrument()-- for span in spans_of(exporter):- assert span.attributes[SpanAttributes.INPUT_VALUE] == "__REDACTED__"- assert span.attributes[SpanAttributes.OUTPUT_VALUE] == "__REDACTED__"- assert SpanAttributes.INPUT_MIME_TYPE not in span.attributes- assert SpanAttributes.OUTPUT_MIME_TYPE not in span.attributes---def test_session_and_user_metadata_survive_on_agent_span(exporter):- agent = make_agent(sync_client())-- with using_session("session-1"), using_metadata({"tenant": "acme"}):- agent.run(QuestionSchema(question="?"))-- (span,) = spans_of(exporter)- assert span.attributes[SpanAttributes.SESSION_ID] == "session-1"- assert json.loads(span.attributes[SpanAttributes.METADATA]) == {"tenant": "acme"}---def test_second_instrument_with_other_settings_warns_and_keeps_first(exporter, caplog):- other_provider = TracerProvider()-- with caplog.at_level(logging.WARNING):- AtomicAgentsInstrumentor().instrument(tracer_provider=other_provider)-- assert "already installed" in caplog.text- make_agent(sync_client()).run(QuestionSchema(question="?"))- assert len(spans_of(exporter)) == 1---def test_uninstrument_restores_originals_and_stops_tracing():- original_run = AtomicAgent.__dict__["run"]- original_tool_run = CalculatorTool.__dict__["run"]- original_init_subclass = BaseTool.__dict__["__init_subclass__"]- exporter = InMemorySpanExporter()- provider = TracerProvider()- provider.add_span_processor(SimpleSpanProcessor(exporter))- instrumentor = AtomicAgentsInstrumentor()-- instrumentor.instrument(tracer_provider=provider)- assert instrumentor.is_instrumented_by_opentelemetry- instrumentor.uninstrument()- instrumentor.uninstrument()-- assert not instrumentor.is_instrumented_by_opentelemetry- assert AtomicAgent.__dict__["run"] is original_run- assert CalculatorTool.__dict__["run"] is original_tool_run- assert BaseTool.__dict__["__init_subclass__"] is original_init_subclass- assert "run" not in MixinCalculatorTool.__dict__-- class AfterwardsTool(BaseTool[CalculationInputSchema, CalculationOutputSchema]):- def run(self, params: CalculationInputSchema) -> CalculationOutputSchema:- return CalculationOutputSchema(result="plain")-- assert AfterwardsTool().input_schema is CalculationInputSchema- make_agent(sync_client()).run(QuestionSchema(question="?"))- AfterwardsTool().run(CalculationInputSchema(expression="x"))- assert spans_of(exporter) == []---def test_phoenix_entry_point_loads_and_instruments_with_only_a_tracer_provider():- (entry_point,) = [point for point in entry_points(group="openinference_instrumentor") if point.name == "atomic_agents"]- exporter = InMemorySpanExporter()- provider = TracerProvider()- provider.add_span_processor(SimpleSpanProcessor(exporter))-- instrumentor = entry_point.load()()- instrumentor.instrument(tracer_provider=provider)- try:- make_agent(sync_client()).run(QuestionSchema(question="?"))- finally:- instrumentor.uninstrument()-- assert isinstance(instrumentor, AtomicAgentsInstrumentor)- assert len(spans_of(exporter)) == 1--[email protected]("options", [{}, {"skip_dep_check": False}])-def test_instrument_without_openinference_raises_install_hint(monkeypatch: pytest.MonkeyPatch, options: dict[str, object]):- monkeypatch.setattr(observability, "_openinference_is_installed", lambda: False)-- with pytest.raises(ImportError, match="atomic-agents\\[observability\\]"):- AtomicAgentsInstrumentor().instrument(**options)---def test_uninstrument_without_openinference_is_a_no_op(monkeypatch):- monkeypatch.setattr(observability, "_openinference_is_installed", lambda: False)-- AtomicAgentsInstrumentor().uninstrument()-- assert not AtomicAgentsInstrumentor().is_instrumented_by_opentelemetry---def test_openinference_below_the_floor_disables_tracing_with_a_warning(monkeypatch, caplog):- monkeypatch.setattr(observability, "version", lambda distribution: "0.1.20")-- with caplog.at_level(logging.WARNING):- AtomicAgentsInstrumentor().instrument()-- assert "0.1.38 or newer is required" in caplog.text- assert not AtomicAgentsInstrumentor().is_instrumented_by_opentelemetry- assert AtomicAgent.__dict__["run"].__name__ == "run"- assert not hasattr(AtomicAgent.__dict__["run"], tracing.TRACED_MARKER)---class CountingProvider(BaseDynamicContextProvider):- def __init__(self):- super().__init__(title="Counter")- self.calls = 0-- def get_info(self) -> str:- self.calls += 1- return f"call {self.calls}"---def test_prompt_hash_identifies_static_sections_despite_dynamic_providers(exporter):- generator = SystemPromptGenerator(background=["v1"], context_providers={"counter": CountingProvider()})- agent = AtomicAgent[QuestionSchema, AnswerSchema](- AgentConfig(client=sync_client(), model="m", system_prompt_generator=generator)- )-- agent.run(QuestionSchema(question="a"))- agent.run(QuestionSchema(question="b"))-- first, second = spans_of(exporter)- assert first.attributes[tracing.ATTR_SYSTEM_PROMPT_SHA256] == second.attributes[tracing.ATTR_SYSTEM_PROMPT_SHA256]- assert first.attributes[tracing.ATTR_CONTEXT_PROVIDERS] == ("counter",)---def test_prompt_hash_changes_when_static_sections_change(exporter):- agent = make_agent(sync_client())- agent.run(QuestionSchema(question="a"))- agent.system_prompt_generator.background = ["v2"]-- agent.run(QuestionSchema(question="b"))-- first, second = spans_of(exporter)- assert first.attributes[tracing.ATTR_SYSTEM_PROMPT_SHA256] != second.attributes[tracing.ATTR_SYSTEM_PROMPT_SHA256]---def test_run_stream_dropped_without_close_still_ends_span(exporter):- agent = make_agent(sync_client())- stream = agent.run_stream(QuestionSchema(question="?"))- next(stream)-- del stream- gc.collect()-- (span,) = spans_of(exporter)- assert span.attributes[tracing.ATTR_STREAM_CLOSED_EARLY] is True---async def test_run_async_stream_cancelled_mid_await_is_not_an_error(exporter):- release = asyncio.Event()-- async def create_partial(**kwargs):- yield AnswerSchema(answer="4")- await release.wait()- yield AnswerSchema(answer="42")-- client = async_client()- client.chat.completions.create_partial = create_partial- agent = make_agent(client)-- async def consume():- async for _ in agent.run_async_stream(QuestionSchema(question="?")):- pass-- task = asyncio.create_task(consume())- await asyncio.sleep(0.01)- task.cancel()- with pytest.raises(asyncio.CancelledError):- await task-- (span,) = spans_of(exporter)- assert span.attributes[tracing.ATTR_STREAM_CLOSED_EARLY] is True- assert output_json(span) == {"answer": "4"}- assert span.status.status_code != StatusCode.ERROR---def test_forge_style_thread_pool_tool_emits_one_span_in_the_callers_trace(exporter, provider_tracer):- class ForgeStyleTool(BaseTool[CalculationInputSchema, CalculationOutputSchema]):- def run(self, params: CalculationInputSchema) -> CalculationOutputSchema:- with ThreadPoolExecutor() as executor:- return executor.submit(contextvars.copy_context().run, asyncio.run, self.run_async(params)).result()-- async def run_async(self, params: CalculationInputSchema) -> CalculationOutputSchema:- return CalculationOutputSchema(result="pooled")-- with provider_tracer.start_as_current_span("root") as root:- ForgeStyleTool().run(CalculationInputSchema(expression="x"))-- tool_span, root_span = spans_of(exporter)- assert tool_span.name == "ForgeStyleTool"- assert tool_span.parent.span_id == root.get_span_context().span_id- assert tool_span.context.trace_id == root_span.context.trace_id---def test_mcp_style_generated_tool_class_emits_one_span(exporter):- async def arun(self, params):- return CalculationOutputSchema(result="mcp")-- def run(self, params):- return asyncio.run(self.arun(params))-- GeneratedTool = types.new_class(- "GeneratedTool",- (BaseTool[CalculationInputSchema, CalculationOutputSchema],),- {},- lambda namespace: namespace.update({"run": run, "arun": arun}),- )-- GeneratedTool().run(CalculationInputSchema(expression="x"))-- (span,) = spans_of(exporter)- assert span.name == "GeneratedTool"- assert output_json(span) == {"result": "mcp"}---def test_async_generator_tool_method_is_left_alone(exporter):- class StreamingTool(BaseTool[CalculationInputSchema, CalculationOutputSchema]):- def run(self, params: CalculationInputSchema) -> CalculationOutputSchema:- return CalculationOutputSchema(result="x")-- async def run_async(self, params: CalculationInputSchema):- yield CalculationOutputSchema(result="chunk")-- assert not hasattr(StreamingTool.__dict__["run_async"], tracing.TRACED_MARKER)- assert hasattr(StreamingTool.__dict__["run"], tracing.TRACED_MARKER)---def test_instrument_uninstrument_instrument_cycle_traces_tools_defined_between_cycles():- exporter = InMemorySpanExporter()- provider = TracerProvider()- provider.add_span_processor(SimpleSpanProcessor(exporter))- instrumentor = AtomicAgentsInstrumentor()- instrumentor.instrument(tracer_provider=provider)- instrumentor.uninstrument()-- class BetweenCyclesTool(BaseTool[CalculationInputSchema, CalculationOutputSchema]):- def run(self, params: CalculationInputSchema) -> CalculationOutputSchema:- return CalculationOutputSchema(result="between")-- original_run = BetweenCyclesTool.__dict__["run"]- instrumentor.instrument(tracer_provider=provider)- try:- BetweenCyclesTool().run(CalculationInputSchema(expression="x"))- finally:- instrumentor.uninstrument()-- assert [span.name for span in spans_of(exporter)] == ["BetweenCyclesTool"]- assert BetweenCyclesTool.__dict__["run"] is original_rundiff --git a/atomic-agents/tests/observability/test_opentelemetry_discovery.py b/atomic-agents/tests/observability/test_opentelemetry_discovery.pyindex 6c40f55..b201c1f 100644--- a/atomic-agents/tests/observability/test_opentelemetry_discovery.py+++ b/atomic-agents/tests/observability/test_opentelemetry_discovery.py@@ -20,7 +20,7 @@ from opentelemetry.util.types import AttributeValueimport atomic_agents.observability as observabilityfrom atomic_agents.observability import AtomicAgentsInstrumentor-from test_openinference_tracing import CalculationInputSchema, CalculatorTool, QuestionSchema, make_agent, sync_client+from test_tracing import CalculationInputSchema, CalculatorTool, QuestionSchema, make_agent, sync_client@pytest.fixture@@ -72,6 +72,7 @@ def test_installed_opentelemetry_entry_point_activates_real_loader(def test_unknown_launcher_options_preserve_provider_and_privacy(local_tracing: tuple[TracerProvider, InMemorySpanExporter],+ caplog: pytest.LogCaptureFixture,) -> None:provider, exporter = local_tracingAtomicAgentsInstrumentor().instrument(@@ -82,13 +83,16 @@ def test_unknown_launcher_options_preserve_provider_and_privacy()run_agent_and_tool()+ assert "custom_launcher_option" in caplog.text+ assert any(record.levelno == logging.WARNING for record in caplog.records)+ assert "skip_dep_check" not in caplog.textspans = exporter.get_finished_spans()assert len(spans) == 2assert [exported_attributes(span)[SpanAttributes.INPUT_VALUE] for span in spans] == ["__REDACTED__", "__REDACTED__"]assert [exported_attributes(span)[SpanAttributes.OUTPUT_VALUE] for span in spans] == ["__REDACTED__", "__REDACTED__"]-def test_loader_skips_missing_openinference_and_continues_without_error(+def test_loader_skips_missing_opentelemetry_and_continues_without_error(monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture) -> None:instrumentor = installed_entry_point("opentelemetry_instrumentor")@@ -98,20 +102,23 @@ def test_loader_skips_missing_openinference_and_continues_without_error(sentinel.name = "sentinel"points = (instrumentor, sentinel)monkeypatch.setattr(_load, "entry_points", lambda group: points if group == "opentelemetry_instrumentor" else ())- monkeypatch.setattr(observability, "_openinference_is_installed", lambda: False)+ monkeypatch.setattr(observability, "_opentelemetry_is_installed", lambda: False)monkeypatch.delenv("OTEL_PYTHON_DISABLED_INSTRUMENTATIONS", raising=False)with caplog.at_level(logging.DEBUG):_load._load_instrumentors(DefaultDistro())sentinel.load.return_value.return_value.instrument.assert_called_once_with(skip_dep_check=True)- assert "atomic-agents[observability]" in caplog.text+ assert (+ 'Atomic Agents tracing needs opentelemetry-api. Install it with: pip install "atomic-agents[observability]" '+ '(add "atomic-agents[phoenix]" for Arize Phoenix / OpenInference attributes)'+ ) in caplog.textassert not [record for record in caplog.records if record.levelno >= logging.ERROR]assert not AtomicAgentsInstrumentor().is_instrumented_by_opentelemetry@pytest.mark.parametrize("first_group", ["openinference_instrumentor", "opentelemetry_instrumentor"])-def test_both_entry_points_keep_first_provider_and_config(+def test_both_entry_points_replace_provider_and_config(first_group: str, local_tracing: tuple[TracerProvider, InMemorySpanExporter], caplog: pytest.LogCaptureFixture) -> None:provider, exporter = local_tracing@@ -129,12 +136,11 @@ def test_both_entry_points_keep_first_provider_and_config(with caplog.at_level(logging.WARNING):second.instrument(tracer_provider=other_provider, config=TraceConfig(hide_inputs=False), skip_dep_check=True)run_agent_and_tool()- assert "already installed" in caplog.text- assert [exported_attributes(span)[SpanAttributes.INPUT_VALUE] for span in exporter.get_finished_spans()] == [- "__REDACTED__",- "__REDACTED__",- ]- assert other_exporter.get_finished_spans() == ()+ assert "tracer provider/config replaced" in caplog.text+ assert exporter.get_finished_spans() == ()+ spans = other_exporter.get_finished_spans()+ assert len(spans) == 2+ assert all(exported_attributes(span)[SpanAttributes.INPUT_VALUE] != "__REDACTED__" for span in spans)finally:second.uninstrument()other_provider.shutdown()@@ -223,7 +229,7 @@ sys.modules['openinference'] = Nonesys.modules['opentelemetry'] = Noneimport atomic_agentsimport atomic_agents.observability-assert 'atomic_agents.observability.openinference_tracing' not in sys.modules+assert 'atomic_agents.observability.tracing' not in sys.modulesprint('optional-free import passed')"""result = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True, timeout=45)diff --git a/atomic-agents/tests/observability/test_tracing.py b/atomic-agents/tests/observability/test_tracing.pynew file mode 100644index 0000000..96d0c8e--- /dev/null+++ b/atomic-agents/tests/observability/test_tracing.py@@ -0,0 +1,1506 @@+import abc+import asyncio+import contextvars+import gc+import json+import logging+import types+from concurrent.futures import ThreadPoolExecutor+from importlib.metadata import entry_points+from typing import Dict, List+from unittest.mock import Mock, PropertyMock++import instructor+import pytest+from openinference.instrumentation import OITracer, TraceConfig, using_metadata, using_session+from openinference.semconv.trace import OpenInferenceSpanKindValues, SpanAttributes+from opentelemetry import trace as trace_api+from opentelemetry.sdk.trace import ReadableSpan, TracerProvider+from opentelemetry.sdk.trace.export import SimpleSpanProcessor+from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter+from opentelemetry.trace import StatusCode+from pydantic import Field, ValidationError++from atomic_agents import AgentConfig, AtomicAgent, BaseIOSchema, BaseTool+from atomic_agents.context import BaseDynamicContextProvider, SystemPromptGenerator+from atomic_agents.observability import AtomicAgentsInstrumentor+import atomic_agents.observability as observability+from atomic_agents.observability import agent_spans, installation, span_attributes, span_payloads, tracing as tracing+++class QuestionSchema(BaseIOSchema):+ """A question for the agent."""++ question: str = Field(...)+++class AnswerSchema(BaseIOSchema):+ """An answer from the agent."""++ answer: str = Field(...)+++class CalculationInputSchema(BaseIOSchema):+ """Expression to evaluate."""++ expression: str = Field(...)+++class CalculationOutputSchema(BaseIOSchema):+ """Result of the evaluation."""++ result: str = Field(...)+++class CalculatorTool(BaseTool[CalculationInputSchema, CalculationOutputSchema]):+ def run(self, params: CalculationInputSchema, precision: int = 2) -> CalculationOutputSchema:+ return CalculationOutputSchema(result=f"{len(params.expression):.{precision}f}")+++class AsyncCalculatorTool(BaseTool[CalculationInputSchema, CalculationOutputSchema]):+ def run(self, params: CalculationInputSchema) -> CalculationOutputSchema:+ return asyncio.run(self.arun(params))++ async def arun(self, params: CalculationInputSchema) -> CalculationOutputSchema:+ return CalculationOutputSchema(result=str(len(params.expression)))+++class ThreadedCalculatorTool(BaseTool[CalculationInputSchema, CalculationOutputSchema]):+ def run(self, params: CalculationInputSchema) -> CalculationOutputSchema:+ return CalculationOutputSchema(result=str(len(params.expression)))++ async def run_async(self, params: CalculationInputSchema) -> CalculationOutputSchema:+ return await asyncio.to_thread(self.run, params)+++class RunMixin:+ def run(self, params: CalculationInputSchema) -> CalculationOutputSchema:+ return CalculationOutputSchema(result="mixin")+++class MixinCalculatorTool(RunMixin, BaseTool[CalculationInputSchema, CalculationOutputSchema]):+ pass+++class StaticRunTool(BaseTool[CalculationInputSchema, CalculationOutputSchema]):+ @staticmethod+ def run(params: CalculationInputSchema) -> CalculationOutputSchema:+ return CalculationOutputSchema(result="static")+++class FailingTool(BaseTool[CalculationInputSchema, CalculationOutputSchema]):+ def run(self, params: CalculationInputSchema) -> CalculationOutputSchema:+ raise ValueError("boom")+++class AnswerStore:+ """Container object pydantic cannot serialize to JSON."""++ def __init__(self):+ self.values: List[int] = []+++class UnserializableInputSchema(BaseIOSchema):+ """Input carrying an arbitrary object."""++ model_config = {"arbitrary_types_allowed": True}+ store: AnswerStore = Field(...)+++class EchoTool(BaseTool[UnserializableInputSchema, CalculationOutputSchema]):+ def run(self, params: UnserializableInputSchema) -> CalculationOutputSchema:+ return CalculationOutputSchema(result="echo")+++def exporter():+ exporter = InMemorySpanExporter()+ provider = TracerProvider()+ provider.add_span_processor(SimpleSpanProcessor(exporter))+ exporter.provider = provider+ instrumentor = AtomicAgentsInstrumentor()+ instrumentor.instrument(tracer_provider=provider)+ yield exporter+ instrumentor.uninstrument()+++def provider_tracer(exporter):+ return exporter.provider.get_tracer("test")+++def sync_client(response=None, partials=None):+ client = Mock(spec=instructor.Instructor)+ client.chat = Mock()+ client.chat.completions = Mock()+ client.chat.completions.create = Mock(return_value=response or AnswerSchema(answer="42"))++ def create_partial(**kwargs):+ for partial in partials or [AnswerSchema(answer="4"), AnswerSchema(answer="42")]:+ yield partial++ client.chat.completions.create_partial = create_partial+ return client+++def async_client(response=None, partials=None):+ client = Mock(spec=instructor.AsyncInstructor)+ client.chat = Mock()+ client.chat.completions = Mock()++ async def create(**kwargs):+ return response or AnswerSchema(answer="42")++ async def create_partial(**kwargs):+ for partial in partials or [AnswerSchema(answer="4"), AnswerSchema(answer="42")]:+ yield partial++ client.chat.completions.create = create+ client.chat.completions.create_partial = create_partial+ return client+++def make_agent(client) -> AtomicAgent[QuestionSchema, AnswerSchema]:+ return AtomicAgent[QuestionSchema, AnswerSchema](AgentConfig(client=client, model="test-model"))+++def spans_of(exporter) -> List[ReadableSpan]:+ return list(exporter.get_finished_spans())+++def kind_of(span: ReadableSpan) -> str:+ return span.attributes[SpanAttributes.OPENINFERENCE_SPAN_KIND]+++def input_json(span: ReadableSpan) -> Dict:+ return json.loads(span.attributes[SpanAttributes.INPUT_VALUE])+++def output_json(span: ReadableSpan) -> Dict:+ return json.loads(span.attributes[SpanAttributes.OUTPUT_VALUE])+++def test_run_emits_agent_span_with_io_and_framework_attributes(exporter):+ agent = make_agent(sync_client())++ response = agent.run(QuestionSchema(question="6*7?"))++ assert response.answer == "42"+ (span,) = spans_of(exporter)+ assert span.name == "invoke_agent AtomicAgent"+ assert kind_of(span) == OpenInferenceSpanKindValues.AGENT.value+ assert span.attributes[SpanAttributes.AGENT_NAME] == "AtomicAgent"+ assert span.attributes[span_attributes.ATTR_AGENT_CLASS] == "AtomicAgent[QuestionSchema, AnswerSchema]"+ assert input_json(span) == {"question": "6*7?"}+ assert output_json(span) == {"answer": "42"}+ assert span.attributes[SpanAttributes.INPUT_MIME_TYPE] == "application/json"+ assert span.attributes[SpanAttributes.OUTPUT_MIME_TYPE] == "application/json"+ assert span.attributes[span_attributes.ATTR_MODEL] == "test-model"+ assert span.attributes[span_attributes.ATTR_MODE] == agent.mode.value+ assert span.attributes[span_attributes.ATTR_INPUT_SCHEMA] == "QuestionSchema"+ assert span.attributes[span_attributes.ATTR_OUTPUT_SCHEMA] == "AnswerSchema"+ assert span.attributes[span_attributes.ATTR_HISTORY_MESSAGE_COUNT] == 0+ assert span.attributes[span_attributes.ATTR_CONTEXT_PROVIDERS] == ()+ assert len(span.attributes[span_attributes.ATTR_SYSTEM_PROMPT_SHA256]) == 64+ assert span.status.status_code == StatusCode.OK+++def test_second_turn_reports_history_size_entering_the_turn(exporter):+ agent = make_agent(sync_client())+ agent.run(QuestionSchema(question="first"))++ agent.run(QuestionSchema(question="second"))++ first, second = spans_of(exporter)+ assert first.attributes[span_attributes.ATTR_HISTORY_MESSAGE_COUNT] == 0+ assert second.attributes[span_attributes.ATTR_HISTORY_MESSAGE_COUNT] == 2+++def test_agent_without_system_role_has_no_prompt_hash(exporter):+ agent = AtomicAgent[QuestionSchema, AnswerSchema](AgentConfig(client=sync_client(), model="m", system_role=None))++ agent.run(QuestionSchema(question="?"))++ (span,) = spans_of(exporter)+ assert span_attributes.ATTR_SYSTEM_PROMPT_SHA256 not in span.attributes+++def test_abstract_intermediate_tool_base_is_not_wrapped(exporter):+ class AbstractCalculator(BaseTool[CalculationInputSchema, CalculationOutputSchema]):+ @abc.abstractmethod+ def run(self, params: CalculationInputSchema) -> CalculationOutputSchema:+ raise NotImplementedError++ assert not hasattr(AbstractCalculator.__dict__["run"], installation.TRACED_MARKER)+++def test_run_without_input_sets_no_input_attribute(exporter):+ agent = make_agent(sync_client())++ agent.run()++ (span,) = spans_of(exporter)+ assert SpanAttributes.INPUT_VALUE not in span.attributes+ assert output_json(span) == {"answer": "42"}+++def test_run_failure_propagates_and_marks_span_error(exporter):+ client = sync_client()+ client.chat.completions.create = Mock(side_effect=RuntimeError("provider down"))+ agent = make_agent(client)++ with pytest.raises(RuntimeError, match="provider down"):+ agent.run(QuestionSchema(question="?"))++ (span,) = spans_of(exporter)+ assert span.status.status_code == StatusCode.ERROR+ assert [event.name for event in span.events] == ["exception"]+++def test_subclassed_agent_is_named_after_the_subclass_and_emits_one_span(exporter):+ class ResearchAgent(AtomicAgent[QuestionSchema, AnswerSchema]):+ def run(self, user_input=None):+ return super().run(user_input)++ agent = ResearchAgent(AgentConfig(client=sync_client(), model="test-model"))++ agent.run(QuestionSchema(question="?"))++ (span,) = spans_of(exporter)+ assert span.name == "invoke_agent ResearchAgent"+++def test_agent_span_nests_under_a_user_chain_span(exporter, provider_tracer):+ agent = make_agent(sync_client())++ with provider_tracer.start_as_current_span("turn") as turn:+ agent.run(QuestionSchema(question="?"))++ agent_span, turn_span = spans_of(exporter)+ assert agent_span.parent.span_id == turn.get_span_context().span_id+ assert turn_span.name == "turn"+++def test_manual_setup_from_the_guide_groups_agents_and_tools_under_one_chain_span():+ exporter = InMemorySpanExporter()+ tracer_provider = TracerProvider()+ tracer_provider.add_span_processor(SimpleSpanProcessor(exporter))+ trace_config = TraceConfig()+ instrumentor = AtomicAgentsInstrumentor()+ instrumentor.instrument(tracer_provider=tracer_provider, config=trace_config)+ tracer = OITracer(tracer_provider.get_tracer(__name__), config=trace_config)+ planner = make_agent(sync_client())+ answerer = make_agent(sync_client())+ calculator = CalculatorTool()+ try:+ with tracer.start_as_current_span("answer_question", openinference_span_kind="chain") as turn:+ turn.set_input("what is 2 + 2?")+ plan = planner.run(QuestionSchema(question="what is 2 + 2?"))+ calculation = calculator.run(CalculationInputSchema(expression=plan.answer))+ answer = answerer.run(QuestionSchema(question=str(calculation.result)))+ turn.set_output(answer.answer)+ finally:+ instrumentor.uninstrument()++ spans = spans_of(exporter)+ chain_span = next(span for span in spans if kind_of(span) == OpenInferenceSpanKindValues.CHAIN.value)+ children = [span for span in spans if span is not chain_span]+ assert [kind_of(span) for span in children] == ["AGENT", "TOOL", "AGENT"]+ assert all(span.parent.span_id == chain_span.get_span_context().span_id for span in children)+ assert chain_span.attributes[SpanAttributes.INPUT_VALUE] == "what is 2 + 2?"+ assert chain_span.attributes[SpanAttributes.OUTPUT_VALUE] == answer.answer+++def test_llm_call_inside_run_becomes_child_of_agent_span(exporter, provider_tracer):+ def create(**kwargs):+ with provider_tracer.start_as_current_span("llm"):+ return AnswerSchema(answer="42")++ client = sync_client()+ client.chat.completions.create = create+ agent = make_agent(client)++ agent.run(QuestionSchema(question="?"))++ llm_span, agent_span = spans_of(exporter)+ assert llm_span.parent.span_id == agent_span.context.span_id+++def test_run_stream_records_final_output_and_keeps_child_context_per_step(exporter, provider_tracer):+ current_between_yields = []++ def create_partial(**kwargs):+ yield AnswerSchema(answer="4")+ with provider_tracer.start_as_current_span("llm-chunk"):+ yield AnswerSchema(answer="42")++ client = sync_client()+ client.chat.completions.create_partial = create_partial+ agent = make_agent(client)++ partials = []+ for partial in agent.run_stream(QuestionSchema(question="?")):+ partials.append(partial.answer)+ current_between_yields.append(trace_api.get_current_span())++ assert partials == ["4", "42"]+ assert all(span is trace_api.INVALID_SPAN for span in current_between_yields)+ chunk_span, agent_span = spans_of(exporter)+ assert agent_span.name == "invoke_agent AtomicAgent"+ assert chunk_span.parent.span_id == agent_span.context.span_id+ assert output_json(agent_span) == {"answer": "42"}+ assert agent_span.status.status_code == StatusCode.OK+++def test_run_stream_preserves_the_generator_return_value(exporter):+ agent = make_agent(sync_client())++ def consume():+ final = yield from agent.run_stream(QuestionSchema(question="?"))+ return final++ generator = consume()+ with pytest.raises(StopIteration) as stop:+ while True:+ next(generator)++ assert stop.value.value.answer == "42"+++def test_run_stream_with_no_partials_ends_ok_without_output(exporter):+ client = sync_client()+ client.chat.completions.create_partial = lambda **kwargs: iter([])+ agent = make_agent(client)++ assert list(agent.run_stream(QuestionSchema(question="?"))) == []++ (span,) = spans_of(exporter)+ assert span.status.status_code == StatusCode.OK+ assert SpanAttributes.OUTPUT_VALUE not in span.attributes+++def test_run_stream_closed_early_ends_span_and_closes_inner_generator(exporter):+ inner_closed = []++ def create_partial(**kwargs):+ try:+ yield AnswerSchema(answer="4")+ yield AnswerSchema(answer="42")+ finally:+ inner_closed.append(True)++ client = sync_client()+ client.chat.completions.create_partial = create_partial+ agent = make_agent(client)++ for partial in agent.run_stream(QuestionSchema(question="?")):+ break++ assert inner_closed == [True]+ (span,) = spans_of(exporter)+ assert span.attributes[span_attributes.ATTR_STREAM_CLOSED_EARLY] is True+ assert output_json(span) == {"answer": "4"}+ assert span.status.status_code == StatusCode.OK+++def test_run_stream_failure_mid_stream_marks_error(exporter):+ def create_partial(**kwargs):+ yield AnswerSchema(answer="4")+ raise RuntimeError("stream broke")++ client = sync_client()+ client.chat.completions.create_partial = create_partial+ agent = make_agent(client)++ with pytest.raises(RuntimeError, match="stream broke"):+ list(agent.run_stream(QuestionSchema(question="?")))++ (span,) = spans_of(exporter)+ assert span.status.status_code == StatusCode.ERROR+++async def test_run_async_emits_agent_span(exporter):+ agent = make_agent(async_client())++ response = await agent.run_async(QuestionSchema(question="?"))++ assert response.answer == "42"+ (span,) = spans_of(exporter)+ assert span.name == "invoke_agent AtomicAgent"+ assert output_json(span) == {"answer": "42"}+ assert span.status.status_code == StatusCode.OK+++async def test_run_async_stream_records_last_partial_and_nests_children(exporter, provider_tracer):+ async def create_partial(**kwargs):+ yield AnswerSchema(answer="4")+ with provider_tracer.start_as_current_span("llm-chunk"):+ yield AnswerSchema(answer="42")++ client = async_client()+ client.chat.completions.create_partial = create_partial+ agent = make_agent(client)++ partials = [partial.answer async for partial in agent.run_async_stream(QuestionSchema(question="?"))]++ assert partials == ["4", "42"]+ chunk_span, agent_span = spans_of(exporter)+ assert agent_span.name == "invoke_agent AtomicAgent"+ assert chunk_span.parent.span_id == agent_span.context.span_id+ assert output_json(agent_span) == {"answer": "42"}+++async def test_run_async_stream_closed_early_ends_span(exporter):+ agent = make_agent(async_client())++ stream = agent.run_async_stream(QuestionSchema(question="?"))+ first = await stream.__anext__()+ await stream.aclose()++ assert first.answer == "4"+ (span,) = spans_of(exporter)+ assert span.attributes[span_attributes.ATTR_STREAM_CLOSED_EARLY] is True+ assert span.status.status_code == StatusCode.OK+++async def test_concurrent_async_runs_keep_separate_spans(exporter):+ first = make_agent(async_client(response=AnswerSchema(answer="one")))+ second = make_agent(async_client(response=AnswerSchema(answer="two")))++ await asyncio.gather(first.run_async(QuestionSchema(question="1")), second.run_async(QuestionSchema(question="2")))++ outputs = {input_json(span)["question"]: output_json(span)["answer"] for span in spans_of(exporter)}+ assert outputs == {"1": "one", "2": "two"}+ assert all(span.parent is None for span in spans_of(exporter))+++def test_tool_defined_before_instrumentation_emits_tool_span(exporter):+ tool = CalculatorTool()++ result = tool.run(CalculationInputSchema(expression="6*7"), precision=1)++ assert result.result == "3.0"+ (span,) = spans_of(exporter)+ assert span.name == "execute_tool CalculationInputSchema"+ assert kind_of(span) == OpenInferenceSpanKindValues.TOOL.value+ assert span.attributes[SpanAttributes.TOOL_NAME] == "CalculationInputSchema"+ assert span.attributes[SpanAttributes.TOOL_DESCRIPTION] == "Expression to evaluate."+ assert json.loads(span.attributes[SpanAttributes.TOOL_PARAMETERS])["title"] == "CalculationInputSchema"+ assert input_json(span) == {"expression": "6*7"}+ assert output_json(span) == {"result": "3.0"}+ assert span.status.status_code == StatusCode.OK+++def test_tool_accepts_keyword_params(exporter):+ result = CalculatorTool().run(params=CalculationInputSchema(expression="1+1"))++ assert result.result == "3.00"+ (span,) = spans_of(exporter)+ assert input_json(span) == {"expression": "1+1"}+++def test_tool_defined_after_instrumentation_emits_tool_span(exporter):+ class LateTool(BaseTool[CalculationInputSchema, CalculationOutputSchema]):+ def run(self, params: CalculationInputSchema) -> CalculationOutputSchema:+ return CalculationOutputSchema(result="late")++ LateTool().run(CalculationInputSchema(expression="x"))++ (span,) = spans_of(exporter)+ assert span.name == "execute_tool CalculationInputSchema"+ assert LateTool().input_schema is CalculationInputSchema+++def test_subclass_calling_super_run_emits_one_span(exporter):+ class LoudCalculatorTool(CalculatorTool):+ def run(self, params: CalculationInputSchema, precision: int = 2) -> CalculationOutputSchema:+ return super().run(params, precision)++ LoudCalculatorTool().run(CalculationInputSchema(expression="2*2"))++ (span,) = spans_of(exporter)+ assert span.name == "execute_tool CalculationInputSchema"+++def test_sync_run_delegating_to_arun_emits_one_span(exporter):+ AsyncCalculatorTool().run(CalculationInputSchema(expression="2*2"))++ (span,) = spans_of(exporter)+ assert span.name == "execute_tool CalculationInputSchema"+ assert output_json(span) == {"result": "3"}+++async def test_run_async_delegating_to_thread_emits_one_span(exporter):+ await ThreadedCalculatorTool().run_async(CalculationInputSchema(expression="3*3"))++ (span,) = spans_of(exporter)+ assert span.name == "execute_tool CalculationInputSchema"+ assert output_json(span) == {"result": "3"}+++def test_mixin_defined_run_is_traced(exporter):+ MixinCalculatorTool().run(CalculationInputSchema(expression="x"))++ (span,) = spans_of(exporter)+ assert span.name == "execute_tool CalculationInputSchema"+++def test_static_run_is_left_alone(exporter):+ assert StaticRunTool.run(CalculationInputSchema(expression="x")).result == "static"+ assert spans_of(exporter) == []+++def test_tool_failure_propagates_same_exception_and_marks_error(exporter):+ with pytest.raises(ValueError, match="boom") as raised:+ FailingTool().run(CalculationInputSchema(expression="x"))++ (span,) = spans_of(exporter)+ assert span.status.status_code == StatusCode.ERROR+ assert raised.value.args == ("boom",)+ assert span.attributes["error.type"] == "ValueError"+ assert [event.name for event in span.events] == ["exception"]+++async def test_tool_run_async_cancellation_leaves_span_unset(exporter):+ started = asyncio.Event()+ release = asyncio.Event()++ class WaitingTool(CalculatorTool):+ async def run_async(self, params: CalculationInputSchema) -> CalculationOutputSchema:+ started.set()+ await release.wait()+ return CalculationOutputSchema(result="never")++ task = asyncio.create_task(WaitingTool().run_async(CalculationInputSchema(expression="x")))+ await asyncio.wait_for(started.wait(), timeout=2)+ task.cancel()+ with pytest.raises(asyncio.CancelledError):+ await task+ (span,) = spans_of(exporter)+ assert span.attributes.get("atomic_agents.cancelled") is True+ assert span.status.status_code == StatusCode.UNSET+ assert span.events == ()+ assert "error.type" not in span.attributes++[email protected]("error_type", [GeneratorExit, asyncio.CancelledError])[email protected]("method_name", ["run", "run_async"])+async def test_tool_cancellation_propagates_without_error(exporter, monkeypatch, error_type, method_name):+ def cancelled_result(*args, **kwargs):+ raise error_type++ monkeypatch.setattr(CalculationOutputSchema, "__init__", cancelled_result)+ tool = ThreadedCalculatorTool()+ with pytest.raises(error_type):+ result = getattr(tool, method_name)(CalculationInputSchema(expression="x"))+ if method_name == "run_async":+ await result+ (span,) = spans_of(exporter)+ assert span.attributes.get("atomic_agents.cancelled") is True+ assert span.status.status_code == StatusCode.UNSET+ assert span.events == ()+ assert "error.type" not in span.attributes+++def test_unserializable_input_still_returns_result(exporter, caplog):+ with caplog.at_level(logging.WARNING):+ result = EchoTool().run(UnserializableInputSchema(store=AnswerStore()))++ assert result.result == "echo"+ (span,) = spans_of(exporter)+ assert span.status.status_code == StatusCode.OK+++def test_agent_run_inside_tool_nests_under_tool_span(exporter):+ agent = make_agent(sync_client())++ class AgentTool(BaseTool[CalculationInputSchema, CalculationOutputSchema]):+ def run(self, params: CalculationInputSchema) -> CalculationOutputSchema:+ return CalculationOutputSchema(result=agent.run(QuestionSchema(question=params.expression)).answer)++ AgentTool().run(CalculationInputSchema(expression="?"))++ agent_span, tool_span = spans_of(exporter)+ assert agent_span.parent.span_id == tool_span.context.span_id+++def test_hide_inputs_and_outputs_redacts_values_and_drops_mime_types():+ exporter = InMemorySpanExporter()+ provider = TracerProvider()+ provider.add_span_processor(SimpleSpanProcessor(exporter))+ instrumentor = AtomicAgentsInstrumentor()+ instrumentor.instrument(tracer_provider=provider, config=TraceConfig(hide_inputs=True, hide_outputs=True))+ try:+ make_agent(sync_client()).run(QuestionSchema(question="secret"))+ CalculatorTool().run(CalculationInputSchema(expression="1+1"))+ finally:+ instrumentor.uninstrument()++ for span in spans_of(exporter):+ assert span.attributes[SpanAttributes.INPUT_VALUE] == "__REDACTED__"+ assert span.attributes[SpanAttributes.OUTPUT_VALUE] == "__REDACTED__"+ assert SpanAttributes.INPUT_MIME_TYPE not in span.attributes+ assert SpanAttributes.OUTPUT_MIME_TYPE not in span.attributes+++def test_session_and_user_metadata_survive_on_agent_span(exporter):+ agent = make_agent(sync_client())++ with using_session("session-1"), using_metadata({"tenant": "acme"}):+ agent.run(QuestionSchema(question="?"))++ (span,) = spans_of(exporter)+ assert span.attributes[SpanAttributes.SESSION_ID] == "session-1"+ assert json.loads(span.attributes[SpanAttributes.METADATA]) == {"tenant": "acme"}+++def test_second_instrument_replaces_provider_and_warns(exporter, caplog):+ other_provider = TracerProvider()+ other_exporter = InMemorySpanExporter()+ other_provider.add_span_processor(SimpleSpanProcessor(other_exporter))+ agent = make_agent(sync_client())+ agent.run(QuestionSchema(question="first"))+ installed = installation._installation+ wrapped_run = AtomicAgent.run++ with caplog.at_level(logging.WARNING):+ AtomicAgentsInstrumentor().instrument(tracer_provider=other_provider)++ agent.run(QuestionSchema(question="second"))+ assert installation._installation is installed+ assert AtomicAgent.run is wrapped_run+ assert len(spans_of(exporter)) == 1+ assert len(spans_of(other_exporter)) == 1+ assert "tracer provider/config replaced" in caplog.text+ assert "in-flight spans finish on the provider they started with" in caplog.text+ assert caplog.records[-1].levelno == logging.WARNING+ other_provider.shutdown()+++def test_uninstrument_restores_originals_and_stops_tracing():+ original_run = AtomicAgent.__dict__["run"]+ original_tool_run = CalculatorTool.__dict__["run"]+ original_init_subclass = BaseTool.__dict__["__init_subclass__"]+ exporter = InMemorySpanExporter()+ provider = TracerProvider()+ provider.add_span_processor(SimpleSpanProcessor(exporter))+ instrumentor = AtomicAgentsInstrumentor()++ instrumentor.instrument(tracer_provider=provider)+ assert instrumentor.is_instrumented_by_opentelemetry+ instrumentor.uninstrument()+ instrumentor.uninstrument()++ assert not instrumentor.is_instrumented_by_opentelemetry+ assert AtomicAgent.__dict__["run"] is original_run+ assert CalculatorTool.__dict__["run"] is original_tool_run+ assert BaseTool.__dict__["__init_subclass__"] is original_init_subclass+ assert "run" not in MixinCalculatorTool.__dict__++ class AfterwardsTool(BaseTool[CalculationInputSchema, CalculationOutputSchema]):+ def run(self, params: CalculationInputSchema) -> CalculationOutputSchema:+ return CalculationOutputSchema(result="plain")++ assert AfterwardsTool().input_schema is CalculationInputSchema+ make_agent(sync_client()).run(QuestionSchema(question="?"))+ AfterwardsTool().run(CalculationInputSchema(expression="x"))+ assert spans_of(exporter) == []+++def test_phoenix_entry_point_loads_and_instruments_with_only_a_tracer_provider():+ (entry_point,) = [point for point in entry_points(group="openinference_instrumentor") if point.name == "atomic_agents"]+ exporter = InMemorySpanExporter()+ provider = TracerProvider()+ provider.add_span_processor(SimpleSpanProcessor(exporter))++ instrumentor = entry_point.load()()+ instrumentor.instrument(tracer_provider=provider)+ try:+ make_agent(sync_client()).run(QuestionSchema(question="?"))+ finally:+ instrumentor.uninstrument()++ assert isinstance(instrumentor, AtomicAgentsInstrumentor)+ assert len(spans_of(exporter)) == 1++[email protected]("options", [{}, {"skip_dep_check": False}])+def test_instrument_without_opentelemetry_raises_install_hint(monkeypatch: pytest.MonkeyPatch, options: dict[str, object]):+ monkeypatch.setattr(observability, "_opentelemetry_is_installed", lambda: False)++ with pytest.raises(ImportError, match="atomic-agents\\[observability\\]"):+ AtomicAgentsInstrumentor().instrument(**options)+++def test_uninstrument_without_opentelemetry_is_a_no_op(monkeypatch):+ monkeypatch.setattr(observability, "_opentelemetry_is_installed", lambda: False)++ AtomicAgentsInstrumentor().uninstrument()++ assert not AtomicAgentsInstrumentor().is_instrumented_by_opentelemetry+++def test_openinference_below_the_floor_only_disables_context_with_a_warning(monkeypatch, caplog):+ monkeypatch.setattr(observability, "version", lambda distribution: "0.1.20")++ with caplog.at_level(logging.WARNING):+ AtomicAgentsInstrumentor().instrument()++ assert "0.1.38 or newer is required" in caplog.text+ assert AtomicAgentsInstrumentor().is_instrumented_by_opentelemetry+ assert span_attributes._context_attributes() == {}+ AtomicAgentsInstrumentor().uninstrument()+++class CountingProvider(BaseDynamicContextProvider):+ def __init__(self):+ super().__init__(title="Counter")+ self.calls = 0++ def get_info(self) -> str:+ self.calls += 1+ return f"call {self.calls}"+++def test_prompt_hash_identifies_static_sections_despite_dynamic_providers(exporter):+ generator = SystemPromptGenerator(background=["v1"], context_providers={"counter": CountingProvider()})+ agent = AtomicAgent[QuestionSchema, AnswerSchema](+ AgentConfig(client=sync_client(), model="m", system_prompt_generator=generator)+ )++ agent.run(QuestionSchema(question="a"))+ agent.run(QuestionSchema(question="b"))++ first, second = spans_of(exporter)+ assert (+ first.attributes[span_attributes.ATTR_SYSTEM_PROMPT_SHA256]+ == second.attributes[span_attributes.ATTR_SYSTEM_PROMPT_SHA256]+ )+ assert first.attributes[span_attributes.ATTR_CONTEXT_PROVIDERS] == ("counter",)+++def test_prompt_hash_changes_when_static_sections_change(exporter):+ agent = make_agent(sync_client())+ agent.run(QuestionSchema(question="a"))+ agent.system_prompt_generator.background = ["v2"]++ agent.run(QuestionSchema(question="b"))++ first, second = spans_of(exporter)+ assert (+ first.attributes[span_attributes.ATTR_SYSTEM_PROMPT_SHA256]+ != second.attributes[span_attributes.ATTR_SYSTEM_PROMPT_SHA256]+ )+++def test_run_stream_dropped_without_close_still_ends_span(exporter):+ agent = make_agent(sync_client())+ stream = agent.run_stream(QuestionSchema(question="?"))+ next(stream)++ del stream+ gc.collect()++ (span,) = spans_of(exporter)+ assert span.attributes[span_attributes.ATTR_STREAM_CLOSED_EARLY] is True+++async def test_run_async_stream_cancelled_mid_await_is_not_an_error(exporter):+ release = asyncio.Event()++ async def create_partial(**kwargs):+ yield AnswerSchema(answer="4")+ await release.wait()+ yield AnswerSchema(answer="42")++ client = async_client()+ client.chat.completions.create_partial = create_partial+ agent = make_agent(client)++ async def consume():+ async for _ in agent.run_async_stream(QuestionSchema(question="?")):+ pass++ task = asyncio.create_task(consume())+ await asyncio.sleep(0.01)+ task.cancel()+ with pytest.raises(asyncio.CancelledError):+ await task++ (span,) = spans_of(exporter)+ assert span.attributes[span_attributes.ATTR_STREAM_CLOSED_EARLY] is True+ assert output_json(span) == {"answer": "4"}+ assert span.status.status_code != StatusCode.ERROR+++def test_forge_style_thread_pool_tool_emits_one_span_in_the_callers_trace(exporter, provider_tracer):+ class ForgeStyleTool(BaseTool[CalculationInputSchema, CalculationOutputSchema]):+ def run(self, params: CalculationInputSchema) -> CalculationOutputSchema:+ with ThreadPoolExecutor() as executor:+ return executor.submit(contextvars.copy_context().run, asyncio.run, self.run_async(params)).result()++ async def run_async(self, params: CalculationInputSchema) -> CalculationOutputSchema:+ return CalculationOutputSchema(result="pooled")++ with provider_tracer.start_as_current_span("root") as root:+ ForgeStyleTool().run(CalculationInputSchema(expression="x"))++ tool_span, root_span = spans_of(exporter)+ assert tool_span.name == "execute_tool CalculationInputSchema"+ assert tool_span.parent.span_id == root.get_span_context().span_id+ assert tool_span.context.trace_id == root_span.context.trace_id+++def test_mcp_style_generated_tool_class_emits_one_span(exporter):+ async def arun(self, params):+ return CalculationOutputSchema(result="mcp")++ def run(self, params):+ return asyncio.run(self.arun(params))++ GeneratedTool = types.new_class(+ "GeneratedTool",+ (BaseTool[CalculationInputSchema, CalculationOutputSchema],),+ {},+ lambda namespace: namespace.update({"run": run, "arun": arun}),+ )++ GeneratedTool().run(CalculationInputSchema(expression="x"))++ (span,) = spans_of(exporter)+ assert span.name == "execute_tool CalculationInputSchema"+ assert output_json(span) == {"result": "mcp"}+++def test_async_generator_tool_method_is_left_alone(exporter):+ class StreamingTool(BaseTool[CalculationInputSchema, CalculationOutputSchema]):+ def run(self, params: CalculationInputSchema) -> CalculationOutputSchema:+ return CalculationOutputSchema(result="x")++ async def run_async(self, params: CalculationInputSchema):+ yield CalculationOutputSchema(result="chunk")++ assert not hasattr(StreamingTool.__dict__["run_async"], installation.TRACED_MARKER)+ assert hasattr(StreamingTool.__dict__["run"], installation.TRACED_MARKER)+++def test_instrument_uninstrument_instrument_cycle_traces_tools_defined_between_cycles():+ exporter = InMemorySpanExporter()+ provider = TracerProvider()+ provider.add_span_processor(SimpleSpanProcessor(exporter))+ instrumentor = AtomicAgentsInstrumentor()+ instrumentor.instrument(tracer_provider=provider)+ instrumentor.uninstrument()++ class BetweenCyclesTool(BaseTool[CalculationInputSchema, CalculationOutputSchema]):+ def run(self, params: CalculationInputSchema) -> CalculationOutputSchema:+ return CalculationOutputSchema(result="between")++ original_run = BetweenCyclesTool.__dict__["run"]+ instrumentor.instrument(tracer_provider=provider)+ try:+ BetweenCyclesTool().run(CalculationInputSchema(expression="x"))+ finally:+ instrumentor.uninstrument()++ assert [span.name for span in spans_of(exporter)] == ["execute_tool CalculationInputSchema"]+ assert BetweenCyclesTool.__dict__["run"] is original_run+++def configured_tracing():+ provider = TracerProvider()+ exporter = InMemorySpanExporter()+ provider.add_span_processor(SimpleSpanProcessor(exporter))+ yield provider, exporter+ AtomicAgentsInstrumentor().uninstrument()+ provider.shutdown()++[email protected]("openinference_enabled", [False, True])+def test_genai_attributes_survive_without_openinference(monkeypatch, configured_tracing, openinference_enabled):+ provider, exporter = configured_tracing+ monkeypatch.setattr(span_attributes, "_import_context_attributes", lambda: None)+ config = tracing.TracingConfig(openinference_attributes=True if openinference_enabled else None)+ AtomicAgentsInstrumentor().instrument(tracer_provider=provider, config=config)+ client = sync_client()+ client.provider = types.SimpleNamespace(value="openai")+ make_agent(client).run(QuestionSchema(question="question"))+ CalculatorTool().run(CalculationInputSchema(expression="1+1"))++ agent_span, tool_span = spans_of(exporter)+ assert [agent_span.name, tool_span.name] == ["invoke_agent AtomicAgent", "execute_tool CalculationInputSchema"]+ assert agent_span.attributes["gen_ai.operation.name"] == "invoke_agent"+ assert agent_span.attributes["gen_ai.agent.name"] == "AtomicAgent"+ assert agent_span.attributes["gen_ai.request.model"] == "test-model"+ assert agent_span.attributes["gen_ai.provider.name"] == "openai"+ assert agent_span.attributes["atomic_agents.agent.class"] == "AtomicAgent[QuestionSchema, AnswerSchema]"+ assert json.loads(agent_span.attributes["atomic_agents.input"]) == {"question": "question"}+ assert json.loads(agent_span.attributes["atomic_agents.output"]) == {"answer": "42"}+ assert tool_span.attributes["gen_ai.operation.name"] == "execute_tool"+ assert tool_span.attributes["gen_ai.tool.name"] == "CalculationInputSchema"+ assert tool_span.attributes["atomic_agents.tool.class"] == "CalculatorTool"+ assert tool_span.attributes["gen_ai.tool.description"] == "Expression to evaluate."+ assert tool_span.attributes["gen_ai.tool.type"] == "function"+ assert json.loads(tool_span.attributes["gen_ai.tool.call.arguments"]) == {"expression": "1+1"}+ assert json.loads(tool_span.attributes["gen_ai.tool.call.result"]) == {"result": "3.00"}+ assert ("openinference.span.kind" in agent_span.attributes) is openinference_enabled+ assert ("openinference.span.kind" in tool_span.attributes) is openinference_enabled+ if openinference_enabled:+ assert agent_span.attributes["input.value"] == agent_span.attributes["atomic_agents.input"]+ assert tool_span.attributes["input.value"] == tool_span.attributes["gen_ai.tool.call.arguments"]+ assert tool_span.attributes["output.value"] == tool_span.attributes["gen_ai.tool.call.result"]+ else:+ assert not ({"input.value", "output.value", "agent.name"} & agent_span.attributes.keys())+ assert not ({"input.value", "output.value", "tool.name", "tool.parameters"} & tool_span.attributes.keys())++[email protected]("config_class", [tracing.TracingConfig, TraceConfig])+def test_both_visibility_configs_redact_primary_payloads(configured_tracing, config_class):+ provider, exporter = configured_tracing+ AtomicAgentsInstrumentor().instrument(tracer_provider=provider, config=config_class(hide_inputs=True, hide_outputs=True))+ make_agent(sync_client()).run(QuestionSchema(question="secret"))+ CalculatorTool().run(CalculationInputSchema(expression="secret"))++ agent_span, tool_span = spans_of(exporter)+ assert agent_span.attributes["atomic_agents.input"] == "__REDACTED__"+ assert agent_span.attributes["atomic_agents.output"] == "__REDACTED__"+ assert tool_span.attributes["gen_ai.tool.call.arguments"] == "__REDACTED__"+ assert tool_span.attributes["gen_ai.tool.call.result"] == "__REDACTED__"++[email protected]("hide_outputs", [False, True])+def test_large_tool_result_is_truncated_unless_hidden(configured_tracing, hide_outputs):+ provider, exporter = configured_tracing+ AtomicAgentsInstrumentor().instrument(+ tracer_provider=provider, config=tracing.TracingConfig(hide_outputs=hide_outputs, openinference_attributes=True)+ )++ class LargeResultTool(BaseTool[CalculationInputSchema, CalculationOutputSchema]):+ def run(self, params: CalculationInputSchema) -> CalculationOutputSchema:+ return CalculationOutputSchema(result="x" * 10240)++ result = LargeResultTool().run(CalculationInputSchema(expression="large"))+ (span,) = spans_of(exporter)+ serialized = result.model_dump_json()+ expected = "__REDACTED__" if hide_outputs else serialized[: 4096 - len("...[truncated]")] + "...[truncated]"+ assert span.attributes["gen_ai.tool.call.result"] == expected+ assert span.attributes["output.value"] == expected+ if hide_outputs:+ assert "gen_ai.tool.call.result.original_length" not in span.attributes+ assert "output.value.original_length" not in span.attributes+ else:+ assert span.attributes["gen_ai.tool.call.result.original_length"] == len(serialized)+ assert span.attributes["output.value.original_length"] == len(serialized)++[email protected]("agent_method", ["run", "run_async", "run_stream", "run_async_stream"])+async def test_suppression_skips_agent_and_tool_spans(exporter, agent_method):+ from opentelemetry import context++ client = async_client() if "async" in agent_method else sync_client()+ agent = make_agent(client)+ token = context.attach(context.set_value("suppress_instrumentation", True))+ try:+ result = getattr(agent, agent_method)(QuestionSchema(question="suppressed"))+ assert await final_agent_answer(agent_method, result) == "42"+ assert CalculatorTool().run(CalculationInputSchema(expression="1+1")).result == "3.00"+ assert (await ThreadedCalculatorTool().run_async(CalculationInputSchema(expression="1+1"))).result == "3"+ finally:+ context.detach(token)+ assert spans_of(exporter) == []+++def test_failures_record_qualified_error_type(exporter):+ client = sync_client()+ client.chat.completions.create.side_effect = ValueError("agent failure")+ with pytest.raises(ValueError, match="agent failure"):+ make_agent(client).run(QuestionSchema(question="fail"))+ with pytest.raises(ValueError, match="boom"):+ FailingTool().run(CalculationInputSchema(expression="fail"))+ for span in spans_of(exporter):+ assert span.attributes["error.type"] == "ValueError"+ assert span.status.status_code == StatusCode.ERROR+ assert len(span.events) == 1+++def test_tracing_config_environment_and_coercion(monkeypatch):+ monkeypatch.setenv("OPENINFERENCE_HIDE_INPUTS", "TrUe")+ monkeypatch.setenv("OPENINFERENCE_HIDE_OUTPUTS", "1")+ monkeypatch.setenv("ATOMIC_AGENTS_TRACE_MAX_ATTRIBUTE_LENGTH", "128")+ from atomic_agents.observability import TracingConfig++ config = TracingConfig.from_environment()+ assert config == TracingConfig(hide_inputs=True, hide_outputs=True, max_attribute_length=128)+ assert TracingConfig.coerce(None) == config+ assert TracingConfig.coerce(config) is config+ assert TracingConfig.coerce(TraceConfig(hide_inputs=True)) == TracingConfig(hide_inputs=True, max_attribute_length=128)+++ "value, expected", [("false", False), ("FaLsE", False), ("0", False), ("true", True), ("1", True), ("", True)]+)+def test_tracing_config_llm_spans_environment(monkeypatch, value, expected):+ from atomic_agents.observability import TracingConfig++ monkeypatch.setenv("ATOMIC_AGENTS_TRACE_LLM_SPANS", value)+ assert TracingConfig.from_environment().llm_spans is expected+ assert TracingConfig.coerce(None).llm_spans is expected+++def test_payload_visibility_preserves_environment_settings(monkeypatch):+ monkeypatch.setenv("ATOMIC_AGENTS_TRACE_LLM_SPANS", "false")+ monkeypatch.setenv("ATOMIC_AGENTS_TRACE_MAX_ATTRIBUTE_LENGTH", "128")+ monkeypatch.setenv("ATOMIC_AGENTS_TRACE_HIDE_OUTPUTS", "true")+ config = tracing.TracingConfig.coerce(TraceConfig(hide_inputs=True, hide_outputs=False))+ assert config.llm_spans is False+ assert config.max_attribute_length == 128+ assert config.hide_inputs is True+ assert config.hide_outputs is False+++def test_tracing_config_llm_spans_defaults_to_enabled(monkeypatch):+ from atomic_agents.observability import TracingConfig++ monkeypatch.delenv("ATOMIC_AGENTS_TRACE_LLM_SPANS", raising=False)+ assert TracingConfig.from_environment().llm_spans is True+ assert TracingConfig().llm_spans is True+++def test_session_is_mirrored_to_genai_conversation(exporter):+ with using_session("session-1"):+ make_agent(sync_client()).run(QuestionSchema(question="?"))+ CalculatorTool().run(CalculationInputSchema(expression="?"))+ for span in spans_of(exporter):+ assert span.attributes["gen_ai.conversation.id"] == "session-1"+++async def final_agent_answer(agent_method, result):+ if agent_method == "run_async":+ return (await result).answer+ elif agent_method == "run_stream":+ return list(result)[-1].answer+ elif agent_method == "run_async_stream":+ return [partial async for partial in result][-1].answer+ else:+ return result.answer+++ "name, expected", [(None, "AtomicAgent"), ("", "AtomicAgent"), ("orchestrator_planning", "orchestrator_planning")]+)+def test_stable_agent_name_is_separate_from_schema_class(exporter, name, expected):+ agent = AtomicAgent[QuestionSchema, AnswerSchema](AgentConfig(client=sync_client(), name=name))+ agent.run(QuestionSchema(question="?"))+ (span,) = spans_of(exporter)+ assert span.name == f"invoke_agent {expected}"+ assert span.attributes["gen_ai.agent.name"] == expected+ assert span.attributes["agent.name"] == expected+ assert span.attributes["atomic_agents.agent.class"] == "AtomicAgent[QuestionSchema, AnswerSchema]"++[email protected]("openinference_enabled", [False, True])+def test_conversation_context_overrides_session_on_agent_and_tool(exporter, openinference_enabled):+ config = tracing.TracingConfig(openinference_attributes=openinference_enabled)+ AtomicAgentsInstrumentor().instrument(tracer_provider=exporter.provider, config=config)+ with using_session("session-1"), observability.using_conversation("c-1"):+ make_agent(sync_client()).run(QuestionSchema(question="?"))+ CalculatorTool().run(CalculationInputSchema(expression="?"))+ agent_span, tool_span = spans_of(exporter)+ assert agent_span.attributes["gen_ai.conversation.id"] == "c-1"+ assert tool_span.attributes["gen_ai.conversation.id"] == "c-1"+ assert agent_span.attributes.get("session.id") == ("c-1" if openinference_enabled else None)+ assert tool_span.attributes.get("session.id") == ("c-1" if openinference_enabled else None)+++def test_session_is_mirrored_even_with_openinference_attributes_disabled(exporter):+ config = tracing.TracingConfig(openinference_attributes=False)+ AtomicAgentsInstrumentor().instrument(tracer_provider=exporter.provider, config=config)+ with using_session("session-1"):+ make_agent(sync_client()).run(QuestionSchema(question="?"))+ CalculatorTool().run(CalculationInputSchema(expression="?"))+ for span in spans_of(exporter):+ assert span.attributes["gen_ai.conversation.id"] == "session-1"+ assert "session.id" not in span.attributes+++def test_conversation_context_restores_after_nested_exception(exporter):+ with observability.using_conversation("outer"):+ with pytest.raises(ValueError), observability.using_conversation("inner"):+ CalculatorTool().run(CalculationInputSchema(expression="inner"))+ raise ValueError("reset context")+ CalculatorTool().run(CalculationInputSchema(expression="outer"))+ CalculatorTool().run(CalculationInputSchema(expression="unset"))+ inner, outer, unset = spans_of(exporter)+ assert inner.attributes["gen_ai.conversation.id"] == "inner"+ assert outer.attributes["gen_ai.conversation.id"] == "outer"+ assert "gen_ai.conversation.id" not in unset.attributes+++async def test_conversation_context_is_task_local(exporter):+ async def run_in_conversation(conversation_id):+ with observability.using_conversation(conversation_id):+ await asyncio.sleep(0)+ await make_agent(async_client()).run_async(QuestionSchema(question="?"))+ CalculatorTool().run(CalculationInputSchema(expression="?"))++ await asyncio.gather(run_in_conversation("first"), run_in_conversation("second"))+ assert [span.attributes["gen_ai.conversation.id"] for span in spans_of(exporter)] == ["first", "first", "second", "second"]+++def test_plain_parent_span_owns_agent_and_tool_trace(exporter):+ with exporter.provider.get_tracer("app").start_as_current_span("turn") as turn:+ make_agent(sync_client()).run(QuestionSchema(question="?"))+ CalculatorTool().run(CalculationInputSchema(expression="?"))+ agent_span, tool_span, parent = spans_of(exporter)+ assert agent_span.context.trace_id == turn.get_span_context().trace_id+ assert tool_span.context.trace_id == turn.get_span_context().trace_id+ assert agent_span.parent.span_id == parent.context.span_id+ assert tool_span.parent.span_id == parent.context.span_id+++def test_same_provider_and_config_is_a_silent_no_op(exporter, caplog):+ installed = installation._installation+ tracer = installed.tracer+ with caplog.at_level(logging.DEBUG, logger=tracing.__name__):+ AtomicAgentsInstrumentor().instrument(tracer_provider=exporter.provider, config=tracing.TracingConfig())+ assert installation._installation is installed+ assert installed.tracer is tracer+ assert not caplog.records+++def test_reinstrument_replaces_config_without_rewrapping(exporter, caplog):+ wrapped_run = AtomicAgent.run+ agent = make_agent(sync_client())+ config = tracing.TracingConfig(hide_inputs=True, openinference_attributes=False)+ with caplog.at_level(logging.WARNING):+ AtomicAgentsInstrumentor().instrument(tracer_provider=exporter.provider, config=config)+ agent.run(QuestionSchema(question="secret"))+ CalculatorTool().run(CalculationInputSchema(expression="secret"))+ agent_span, tool_span = spans_of(exporter)+ assert agent_span.attributes["atomic_agents.input"] == "__REDACTED__"+ assert tool_span.attributes["gen_ai.tool.call.arguments"] == "__REDACTED__"+ assert "agent.name" not in agent_span.attributes+ assert AtomicAgent.run is wrapped_run+ assert "tracer provider/config replaced" in caplog.text+++def test_in_flight_span_finishes_on_original_provider(exporter):+ other_provider = TracerProvider()+ other_exporter = InMemorySpanExporter()+ other_provider.add_span_processor(SimpleSpanProcessor(other_exporter))+ client = sync_client()++ def replace_provider(**kwargs):+ AtomicAgentsInstrumentor().instrument(tracer_provider=other_provider)+ CalculatorTool().run(CalculationInputSchema(expression="?"))+ return AnswerSchema(answer="42")++ client.chat.completions.create.side_effect = replace_provider+ make_agent(client).run(QuestionSchema(question="?"))+ assert len(spans_of(exporter)) == 1+ assert len(spans_of(other_exporter)) == 1+ (agent_span,) = spans_of(exporter)+ (tool_span,) = spans_of(other_exporter)+ assert agent_span.name == "invoke_agent AtomicAgent"+ assert tool_span.parent.span_id == agent_span.context.span_id+ assert tool_span.context.trace_id == agent_span.context.trace_id+ other_provider.shutdown()++[email protected]("name", ["model", "mode"])+def test_none_descriptions_are_omitted_without_otel_warnings(exporter, caplog, name):+ agent = make_agent(sync_client())+ setattr(agent, name, None)+ with agent_spans._agent_span(agent, (), {}):+ pass+ (span,) = spans_of(exporter)+ assert f"atomic_agents.{name}" not in span.attributes+ assert "gen_ai.request.model" not in span.attributes if name == "model" else True+ assert "Invalid type" not in caplog.text+++def test_broken_history_preserves_mandatory_agent_attributes(exporter, monkeypatch, caplog):+ agent = make_agent(sync_client())+ monkeypatch.setattr(type(agent), "history", PropertyMock(side_effect=RuntimeError("history unavailable")), raising=False)+ with caplog.at_level(logging.DEBUG), agent_spans._agent_span(agent, (), {}):+ pass+ (span,) = spans_of(exporter)+ assert span.attributes["gen_ai.operation.name"] == "invoke_agent"+ assert span.attributes["gen_ai.agent.name"] == "AtomicAgent"+ assert span.attributes["atomic_agents.agent.class"] == "AtomicAgent[QuestionSchema, AnswerSchema]"+ assert span.attributes["gen_ai.request.model"] == "test-model"+ assert "atomic_agents.history.message_count" not in span.attributes+ assert sum("history.message_count" in record.message for record in caplog.records) == 1+++def test_broken_tool_description_preserves_other_attributes(exporter, monkeypatch):+ monkeypatch.setattr(CalculatorTool, "tool_description", PropertyMock(side_effect=RuntimeError("unavailable")))+ CalculatorTool().run(CalculationInputSchema(expression="1+1"))+ (span,) = spans_of(exporter)+ assert span.attributes["gen_ai.tool.name"] == "CalculationInputSchema"+ assert span.attributes["atomic_agents.tool.class"] == "CalculatorTool"+ assert span.attributes["tool.name"] == "CalculationInputSchema"+ assert "tool.parameters" in span.attributes+ assert "gen_ai.tool.description" not in span.attributes+++def test_configured_tool_name_is_the_span_join_key(exporter):+ tool = CalculatorTool()+ tool.config.title = "calculate"+ tool.run(CalculationInputSchema(expression="1+1"))+ (span,) = spans_of(exporter)+ assert span.name == "execute_tool calculate"+ assert span.attributes["gen_ai.tool.name"] == span.attributes["tool.name"] == "calculate"+ assert span.attributes["atomic_agents.tool.class"] == "CalculatorTool"++[email protected]("hide", [False, True])+def test_hidden_payload_is_never_serialized(exporter, caplog, hide):+ payload = UnserializableInputSchema(store=AnswerStore())+ with exporter.provider.get_tracer("test").start_as_current_span("payload") as span:+ span_payloads._record_payload(span, (("payload", "payload.mime_type"),), payload, hide, tracing.TracingConfig())+ (span,) = spans_of(exporter)+ assert ("Could not serialize tracing payload" in caplog.text) is not hide+ assert span.attributes.get("payload") == ("__REDACTED__" if hide else None)+ assert "payload.mime_type" not in span.attributes++[email protected]("hide", [False, True])+def test_hidden_non_model_payload_does_not_call_json_dumps(exporter, monkeypatch, caplog, hide):+ serialize = Mock(side_effect=ValueError("serialization failed"))+ monkeypatch.setattr(span_payloads.json, "dumps", serialize)+ with exporter.provider.get_tracer("test").start_as_current_span("payload") as span:+ span_payloads._record_payload(span, (("payload", None),), object(), hide, tracing.TracingConfig())+ assert serialize.call_count == (0 if hide else 1)+ assert ("Could not serialize tracing payload" in caplog.text) is not hide+++ "provider,expected",+ [+ ("azure_openai", "azure.ai.openai"),+ ("vertexai", "gcp.vertex_ai"),+ ("gemini", "gcp.gemini"),+ ("genai", "gcp.gemini"),+ ("bedrock", "aws.bedrock"),+ ("mistral", "mistral_ai"),+ ("xai", "x_ai"),+ ("openai", "openai"),+ ("anthropic", "anthropic"),+ ("cohere", "cohere"),+ ("groq", "groq"),+ ("deepseek", "deepseek"),+ ("perplexity", "perplexity"),+ ("custom", "custom"),+ ],+)+def test_instructor_provider_names_follow_genai_conventions(exporter, provider, expected):+ client = sync_client()+ client.provider = types.SimpleNamespace(value=provider)+ make_agent(client).run(QuestionSchema(question="question"))+ (span,) = spans_of(exporter)+ assert span.attributes["gen_ai.provider.name"] == expected+++def test_none_context_attributes_are_filtered(exporter, monkeypatch, caplog):+ monkeypatch.setattr(span_attributes, "_context_attributes", lambda: {"optional": None})+ make_agent(sync_client()).run(QuestionSchema(question="question"))+ (span,) = spans_of(exporter)+ assert "optional" not in span.attributes+ assert "gen_ai.provider.name" not in span.attributes+ assert "Invalid type" not in caplog.text++[email protected]("prefix", ["ATOMIC_AGENTS_TRACE", "OPENINFERENCE"])+def test_visibility_environment_supports_both_spellings(monkeypatch, prefix):+ monkeypatch.delenv("ATOMIC_AGENTS_TRACE_HIDE_INPUTS", raising=False)+ monkeypatch.delenv("ATOMIC_AGENTS_TRACE_HIDE_OUTPUTS", raising=False)+ monkeypatch.setenv(f"{prefix}_HIDE_INPUTS", "TRUE")+ monkeypatch.setenv(f"{prefix}_HIDE_OUTPUTS", "1")+ config = tracing.TracingConfig.from_environment()+ assert config.hide_inputs is True+ assert config.hide_outputs is True+++def test_atomic_visibility_environment_overrides_openinference(monkeypatch):+ monkeypatch.setenv("OPENINFERENCE_HIDE_INPUTS", "true")+ monkeypatch.setenv("OPENINFERENCE_HIDE_OUTPUTS", "true")+ monkeypatch.setenv("ATOMIC_AGENTS_TRACE_HIDE_INPUTS", "false")+ monkeypatch.setenv("ATOMIC_AGENTS_TRACE_HIDE_OUTPUTS", "")+ config = tracing.TracingConfig.from_environment()+ assert config.hide_inputs is False+ assert config.hide_outputs is False+++def test_error_status_and_recorded_message_are_bounded(configured_tracing):+ provider, exporter = configured_tracing+ AtomicAgentsInstrumentor().instrument(tracer_provider=provider, config=tracing.TracingConfig(max_attribute_length=12))+ error = ValueError("private" * 100)+ agent = make_agent(sync_client())+ with pytest.raises(ValueError) as raised, agent_spans._agent_span(agent, (), {}):+ raise error+ (span,) = spans_of(exporter)+ expected = "...[truncate"+ assert raised.value is error+ assert span.status.description == expected+ assert span.events[0].attributes["exception.message"] == expected+ assert len(span.events[0].attributes["exception.stacktrace"]) <= 12+++def test_hidden_validation_errors_do_not_export_model_output(configured_tracing):+ provider, exporter = configured_tracing+ AtomicAgentsInstrumentor().instrument(tracer_provider=provider, config=tracing.TracingConfig(hide_outputs=True))+ try:+ QuestionSchema.model_validate({"question": ["secret model output"]})+ except ValidationError as error:+ with provider.get_tracer("test").start_as_current_span("failure") as span:+ span_payloads._record_failure(span, error)+ (span,) = spans_of(exporter)+ assert span.status.description == "ValidationError: 1 errors (__REDACTED__)"+ assert span.events[0].attributes["exception.message"] == span.status.description+ assert "secret model output" not in str(span.events[0].attributes)+++def test_tool_parameters_use_the_payload_size_limit(configured_tracing):+ provider, exporter = configured_tracing+ AtomicAgentsInstrumentor().instrument(+ tracer_provider=provider, config=tracing.TracingConfig(max_attribute_length=12, openinference_attributes=True)+ )+ CalculatorTool().run(CalculationInputSchema(expression="x"))+ (span,) = spans_of(exporter)+ parameters = json.dumps(CalculationInputSchema.model_json_schema())+ assert span.attributes["tool.parameters"] == "...[truncate"+ assert span.attributes["tool.parameters.original_length"] == len(parameters)++[email protected]("value, expected", [("", 4096), ("4k", 4096), ("0", 4096), ("-5", 4096), ("512", 512)])[email protected]("visibility", [None, TraceConfig(hide_inputs=True)])+def test_attribute_length_environment_is_tolerant(monkeypatch, caplog, value, expected, visibility):+ monkeypatch.setenv("ATOMIC_AGENTS_TRACE_MAX_ATTRIBUTE_LENGTH", value)+ with caplog.at_level(logging.WARNING):+ config = tracing.TracingConfig.coerce(visibility)+ assert config.max_attribute_length == expected+ assert config.hide_inputs is (visibility is not None)+ warnings = [record for record in caplog.records if record.levelno == logging.WARNING]+ if expected == 4096:+ assert len(warnings) == 1+ assert repr(value) in warnings[0].message+ else:+ assert warnings == []+++def test_reinstall_without_config_preserves_redaction(exporter, caplog):+ config = tracing.TracingConfig(hide_inputs=True, openinference_attributes=False)+ AtomicAgentsInstrumentor().instrument(tracer_provider=exporter.provider, config=config)+ other_provider = TracerProvider()+ other_exporter = InMemorySpanExporter()+ other_provider.add_span_processor(SimpleSpanProcessor(other_exporter))+ caplog.clear()+ try:+ with caplog.at_level(logging.WARNING):+ AtomicAgentsInstrumentor().instrument(tracer_provider=other_provider)+ assert installation._current_installation().config is config+ assert installation._current_installation().tracer_provider is other_provider+ make_agent(sync_client()).run(QuestionSchema(question="secret"))+ assert spans_of(other_exporter)[0].attributes["atomic_agents.input"] == "__REDACTED__"+ assert len(caplog.records) == 1+ assert "replaced" in caplog.records[0].message+ finally:+ other_provider.shutdown()+++def test_reinstall_without_provider_preserves_provider(exporter, caplog):+ config = tracing.TracingConfig(hide_outputs=True)+ with caplog.at_level(logging.WARNING):+ AtomicAgentsInstrumentor().instrument(config=config)+ assert installation._current_installation().tracer_provider is exporter.provider+ caplog.clear()+ AtomicAgentsInstrumentor().instrument()+ assert caplog.records == []+ make_agent(sync_client()).run(QuestionSchema(question="?"))+ assert spans_of(exporter)[0].attributes["atomic_agents.output"] == "__REDACTED__"+++async def test_run_async_cancellation_leaves_span_unset(exporter):+ started = asyncio.Event()+ release = asyncio.Event()++ async def create(**kwargs):+ started.set()+ await release.wait()+ return AnswerSchema(answer="never")++ client = async_client()+ client.chat.completions.create = create+ task = asyncio.create_task(make_agent(client).run_async(QuestionSchema(question="?")))+ await asyncio.wait_for(started.wait(), timeout=2)+ task.cancel()+ with pytest.raises(asyncio.CancelledError):+ await task+ (span,) = spans_of(exporter)+ assert span.attributes.get("atomic_agents.cancelled") is True+ assert span.status.status_code == StatusCode.UNSET+ assert span.events == ()+ assert "error.type" not in span.attributes++[email protected]("error_type", [GeneratorExit, asyncio.CancelledError])+def test_run_cancellation_leaves_span_unset(exporter, error_type):+ client = sync_client()+ client.chat.completions.create.side_effect = error_type+ with pytest.raises(error_type):+ make_agent(client).run(QuestionSchema(question="?"))+ (span,) = spans_of(exporter)+ assert span.attributes.get("atomic_agents.cancelled") is True+ assert span.status.status_code == StatusCode.UNSET+ assert span.events == ()+++def test_run_stream_cancellation_marks_closed_early(exporter):+ def create_partial(**kwargs):+ yield AnswerSchema(answer="4")+ raise asyncio.CancelledError++ client = sync_client()+ client.chat.completions.create_partial = create_partial+ stream = make_agent(client).run_stream(QuestionSchema(question="?"))+ next(stream)+ with pytest.raises(asyncio.CancelledError):+ next(stream)+ (span,) = spans_of(exporter)+ assert span.attributes.get(span_attributes.ATTR_STREAM_CLOSED_EARLY) is True+ assert span.status.status_code == StatusCode.UNSET+ assert span.events == ()++[email protected]("broken_name", [False, True])+def test_agent_name_falls_back_without_initialized_base(exporter, broken_name):+ class UninitializedAgent(AtomicAgent[QuestionSchema, AnswerSchema]):+ def __init__(self):+ self.client = sync_client()++ if broken_name:+ UninitializedAgent.name = property(lambda self: 1 / 0)+ agent = UninitializedAgent()+ try:+ display_name = span_attributes._agent_display_name(agent)+ except (AttributeError, ZeroDivisionError):+ display_name = None+ assert display_name == "UninitializedAgent"+ with agent_spans._agent_span(agent, (), {}):+ pass+ (span,) = spans_of(exporter)+ assert span.name == "invoke_agent UninitializedAgent"+ assert span.attributes["gen_ai.agent.name"] == "UninitializedAgent"diff --git a/atomic-examples/phoenix-tracing/README.md b/atomic-examples/phoenix-tracing/README.mdindex 077801d..24bc3da 100644--- a/atomic-examples/phoenix-tracing/README.md+++ b/atomic-examples/phoenix-tracing/README.md@@ -6,17 +6,30 @@ produces this tree in Phoenix:```CHAIN answer_question-├── AGENT AtomicAgent[BasicChatInputSchema, CalculationPlanSchema].run+├── AGENT invoke_agent {agent name}+│ ├── LLM chat {model} (Atomic Agents attempt span, one per Instructor attempt)│ └── LLM ChatCompletion (from openinference-instrumentation-openai)-├── TOOL CalculatorTool-└── AGENT AtomicAgent[CalculationResultSchema, BasicChatOutputSchema].run+├── TOOL execute_tool {tool name}+└── AGENT invoke_agent {agent name}+ ├── LLM chat {model}└── LLM ChatCompletion```+Running the OpenAI instrumentor next to the Atomic Agents attempt spans gives two LLM layers. To keep one, pass+`TracingConfig(llm_spans=False)` when you call `AtomicAgentsInstrumentor().instrument(...)` yourself.+The AGENT and TOOL spans come from `atomic_agents.observability`, which Phoenix activates automaticallythrough `register(auto_instrument=True)`. The LLM spans come from the OpenAI instrumentor. The CHAIN spanis the only manual tracing in `main.py`; without it every agent and tool call would be its own trace.+## Install++Inside this repository `uv sync` installs everything. In your own project:++```bash+pip install "atomic-agents[phoenix]" arize-phoenix-otel openinference-instrumentation-openai+```+## Run it1. Start Phoenix locally (or point `PHOENIX_COLLECTOR_ENDPOINT` at a hosted instance):diff --git a/atomic-examples/phoenix-tracing/phoenix_tracing/main.py b/atomic-examples/phoenix-tracing/phoenix_tracing/main.pyindex ae1f9a3..13d119c 100644--- a/atomic-examples/phoenix-tracing/phoenix_tracing/main.py+++ b/atomic-examples/phoenix-tracing/phoenix_tracing/main.py@@ -15,6 +15,7 @@ from phoenix_tracing.tools.calculator import CalculatorTool, CalculatorToolInputload_dotenv()+os.environ.setdefault("ATOMIC_AGENTS_TRACE_LLM_SPANS", "false")tracer_provider = register(project_name="atomic-agents-example", auto_instrument=True)tracer = tracer_provider.get_tracer(__name__)console = Console()diff --git a/atomic-examples/phoenix-tracing/pyproject.toml b/atomic-examples/phoenix-tracing/pyproject.tomlindex c626725..e09a5bf 100644--- a/atomic-examples/phoenix-tracing/pyproject.toml+++ b/atomic-examples/phoenix-tracing/pyproject.toml@@ -15,7 +15,7 @@ authors = []requires-python = ">=3.12"dependencies = [- "atomic-agents[observability]",+ "atomic-agents[phoenix]","arize-phoenix-otel>=0.17.0,<1.0.0","openinference-instrumentation-openai>=0.1.30,<0.2.0","instructor>=1.17.0,<2",diff --git a/docs/guides/observability.md b/docs/guides/observability.mdindex 1bcff64..fe47927 100644--- a/docs/guides/observability.md+++ b/docs/guides/observability.md@@ -1,69 +1,148 @@-# Observability with Arize Phoenix and OpenTelemetry+# Observability with OpenTelemetry-Atomic Agents ships an optional [OpenInference](https://github.com/Arize-ai/openinference) instrumentor. Once it-is installed, every `AtomicAgent` run and every `BaseTool` run becomes a span that-[Arize Phoenix](https://github.com/Arize-ai/phoenix), Arize AX, Langfuse, or any other OpenTelemetry backend can-display. Nothing changes for applications that do not opt in.+Atomic Agents can turn every agent run, tool run and LLM attempt into an OpenTelemetry span. The spans follow the+OpenTelemetry GenAI semantic conventions, so they show up on any OpenTelemetry backend: a plain OTLP collector,+Grafana Tempo, Sentry, Jaeger, Honeycomb, Datadog, Arize Phoenix, Langfuse and so on. Only `opentelemetry-api` is+required. Nothing changes for applications that do not opt in.-## What you get+You get three kinds of spans:-| Call | Span | Attributes |-|------|------|------------|-| `agent.run`, `run_async`, `run_stream`, `run_async_stream` | `AGENT`, named `AtomicAgent[InputSchema, OutputSchema].run` (a subclass reports its own name) | `agent.name`, `input.value` / `output.value` as JSON, `atomic_agents.model`, `atomic_agents.mode`, `atomic_agents.input_schema`, `atomic_agents.output_schema`, `atomic_agents.history.message_count` (history size entering the turn), `atomic_agents.context_providers`, `atomic_agents.system_prompt.sha256` (hash of the static prompt sections: background, steps, output instructions) |-| `tool.run`, `run_async`, `arun` on any `BaseTool` subclass, including MCP tools | `TOOL`, named after the tool class | `tool.name`, `tool.description`, `tool.parameters` (the input JSON schema), `input.value` / `output.value` as JSON |+- **AGENT**: one per `AtomicAgent` run (`run`, `run_async`, `run_stream`, `run_async_stream`).+- **TOOL**: one per `BaseTool` run, including MCP tools.+- **LLM attempt**: one per call Instructor makes to the model, so retries after a validation error are visible.-Spans carry `session.id`, `user.id`, `metadata` and tags when you wrap calls in the OpenInference context-managers (`using_session`, `using_user`, `using_metadata`, `using_attributes`). Exceptions set the span status to-`ERROR` and are recorded as span events. Streaming spans record the final response when the stream completes,-or the last partial plus `atomic_agents.stream.closed_early = true` when the consumer stops early or the task-is cancelled (a cancelled stream is not reported as an error).+## Install-LLM spans are not produced by Atomic Agents. Install exactly one LLM instrumentor for the provider SDK you use-(`openinference-instrumentation-openai`, `-anthropic`, `-litellm`, ...). Installing both a provider instrumentor and-`openinference-instrumentation-instructor` gives you two layers of LLM spans. Because the agent span is the current-span while the provider call runs, the LLM span (with prompts, tokens and the model name) nests under it. Instructor-retries show up as several LLM spans under one agent span.+```bash+pip install "atomic-agents[observability]" opentelemetry-sdk opentelemetry-exporter-otlp-proto-http+```-## Install+The `observability` extra installs `opentelemetry-api` and nothing else. You bring the SDK and an exporter for your+backend. The `phoenix` extra adds the OpenInference packages:```bash-pip install "atomic-agents[observability]" arize-phoenix-otel openinference-instrumentation-openai+pip install "atomic-agents[phoenix]"```-The `observability` extra installs `openinference-instrumentation`, `openinference-semantic-conventions` and-`opentelemetry-api`. The Phoenix path below works without the extra because `arize-phoenix-otel` already depends on-those packages; the manual path needs the extra plus an OpenTelemetry SDK, an exporter, and the instrumentor for-your LLM client (the example below uses the OpenAI one):+With those installed, Atomic Agents also writes the OpenInference attribute family (turned on automatically) and the+OpenInference context managers `using_session`, `using_user` and `using_metadata` start to work. Backends that read+OpenInference attributes, such as Arize Phoenix, use them for their own views. Every other backend works from the GenAI+attributes alone.++## Setup with any OpenTelemetry backend (manual)++```python+from opentelemetry.sdk.trace import TracerProvider+from opentelemetry.sdk.trace.export import BatchSpanProcessor+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter++from atomic_agents.observability import AtomicAgentsInstrumentor, TracingConfig++tracer_provider = TracerProvider()+tracer_provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter()))++AtomicAgentsInstrumentor().instrument(tracer_provider=tracer_provider, config=TracingConfig())+```++`OTLPSpanExporter()` reads the standard `OTEL_EXPORTER_OTLP_*` environment variables, so this code stays the same for+every backend below. Only the environment changes. `tracer_provider` and `config` are both optional: without a provider,+the global one is used, and without a config you get the defaults described under "Privacy and size".++Call `instrument()` once, at import time or in your service's startup hook, before any agent runs.++## Backends++The GenAI attributes are the OpenTelemetry standard, and backends with LLM views (Datadog LLM Observability, Langfuse,+Phoenix, Sentry AI monitoring, PostHog) read them. Every recipe below uses the setup above. Only the exporter+configuration differs.++What has actually been run: the span output is tested offline against an in-memory exporter, and the Phoenix recipe is+the [phoenix-tracing example](https://github.com/eigenwise/atomic-agents/tree/main/atomic-examples/phoenix-tracing).+Everything marked "untested" follows the vendor's documented OTLP settings but has not been run against a live account.+Check the vendor's page before relying on it.++**Plain OTLP collector** (untested against a live collector):```bash-pip install "atomic-agents[observability]" opentelemetry-sdk opentelemetry-exporter-otlp-proto-http openinference-instrumentation-openai+export OTEL_SERVICE_NAME=my-app+export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318+export OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf```-## Setup with Phoenix (automatic)+**Grafana Tempo** (untested): send OTLP to Tempo directly or through a collector or Grafana Alloy. Self-hosted, point the+endpoint at Tempo's OTLP HTTP port (4318 by default). Grafana Cloud uses its OTLP gateway plus a basic-auth header built+from your instance ID and an access token:++```bash+export OTEL_EXPORTER_OTLP_ENDPOINT=https://otlp-gateway-<zone>.grafana.net/otlp+export OTEL_EXPORTER_OTLP_HEADERS="Authorization=Basic%20<base64 of instance_id:token>"+```++**Jaeger** (untested): current Jaeger releases accept OTLP directly.++```bash+docker run --rm -p 16686:16686 -p 4318:4318 jaegertracing/all-in-one+export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318+```++**Sentry** (follows Sentry's Python OTLP integration docs, untested here). The integration sets up the exporter itself,+so you do not build a `TracerProvider` and do not set `traces_sample_rate`:++```bash+pip install "atomic-agents[observability]" "sentry-sdk[opentelemetry-otlp]"+``````python-from phoenix.otel import register+import sentry_sdk+from sentry_sdk.integrations.otlp import OTLPIntegration-tracer_provider = register(project_name="my-app", auto_instrument=True)+from atomic_agents.observability import AtomicAgentsInstrumentor++sentry_sdk.init(dsn="https://<key>@o<orgId>.ingest.sentry.io/<projectId>", integrations=[OTLPIntegration()])+AtomicAgentsInstrumentor().instrument()```-`register(auto_instrument=True)` discovers the Atomic Agents instrumentor through the `openinference_instrumentor`-entry point and activates it together with every other OpenInference instrumentor in the environment. Do this once,-at import time or in your service's startup hook, before any agent runs. Phoenix reads `PHOENIX_COLLECTOR_ENDPOINT`,-`PHOENIX_PROJECT` (alias `PHOENIX_PROJECT_NAME`) and `PHOENIX_API_KEY` from the environment. With no endpoint set,-`register()` exports over gRPC to `localhost:4317`; set `PHOENIX_COLLECTOR_ENDPOINT=http://localhost:6006` to send-HTTP to the port the Phoenix UI serves on.+Sentry's OTLP ingest drops span events. The `validation_error` event and recorded exceptions do not arrive there, while+`error.type`, the span status and `atomic_agents.llm.last_validation_error` do. Whether Sentry's AI Agents dashboard+fills from these spans is unverified.-Two things to know about the automatic path:+**Honeycomb** (untested): use `https://api.eu1.honeycomb.io` for the EU region. Honeycomb's Agent Timeline groups by+`gen_ai.conversation.id`, so wrap calls in `using_conversation` (see "Naming and conversations").-- Phoenix imports `atomic_agents` to load the entry point. That import is the same as any `import atomic_agents` in- your application, including the Instructor streaming patch the framework applies at import.-- Existing `auto_instrument=True` users start exporting agent and tool payloads when they upgrade to a version with the- instrumentor. Use the privacy controls below if that is not wanted.+```bash+export OTEL_EXPORTER_OTLP_ENDPOINT=https://api.honeycomb.io+export OTEL_EXPORTER_OTLP_HEADERS="x-honeycomb-team=<api key>"+```++**Datadog** (untested): send OTLP to a Datadog Agent with OTLP ingestion enabled (HTTP on 4318 by default) and set+`OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318`. Datadog LLM Observability maps `invoke_agent` spans to agents,+`execute_tool` to tools and `chat` to LLM calls, and reads the `gen_ai.usage.*` tokens and `gen_ai.conversation.id`.+It reads prompt and response text only from `gen_ai.input.messages` and `gen_ai.output.messages`, which Atomic Agents+does not emit, so you see spans, tokens and sessions but not the agent text. PostHog behaves the same way.++**Arize Phoenix**: see "Setup with Phoenix" below.++**Langfuse** (untested): Langfuse takes OTLP at `/api/public/otel` with basic auth built from your public and secret+key. Its default filter keeps spans that carry `gen_ai.*` attributes, which these spans do.++```bash+export OTEL_EXPORTER_OTLP_ENDPOINT=https://cloud.langfuse.com/api/public/otel+export OTEL_EXPORTER_OTLP_HEADERS="Authorization=Basic%20<base64 of public_key:secret_key>"+```++**PostHog** (untested): PostHog AI Observability ingests OTLP over HTTP only (protobuf or JSON, no gRPC). Use the+`TRACES` variables, because the generic endpoint variable appends `/v1/traces` and gets a 404. Use your project's client+API host in the URL, see [PostHog's OpenTelemetry page](https://posthog.com/docs/ai-observability/installation/opentelemetry).+`chat` spans become `$ai_generation` events and the others become `$ai_span`.++```bash+export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="<client api host>/i/v0/ai/otel"+export OTEL_EXPORTER_OTLP_TRACES_HEADERS="Authorization=Bearer%20<project token>"+```## Setup with the OpenTelemetry launcher (automatic)-Install the optional tracing adapter, launcher, SDK, and exporter explicitly:+Install the tracing adapter, launcher, SDK and exporter explicitly:```bashpip install "atomic-agents[observability]" opentelemetry-distro opentelemetry-exporter-otlp-proto-http@@ -75,115 +154,294 @@ Run your existing application with environment-only configuration:export OTEL_SERVICE_NAME=my-appexport OTEL_TRACES_EXPORTER=otlpexport OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf-export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:6006+export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318export OTEL_METRICS_EXPORTER=noneexport OTEL_LOGS_EXPORTER=none-export OPENINFERENCE_HIDE_INPUTS=true-export OPENINFERENCE_HIDE_OUTPUTS=true+export ATOMIC_AGENTS_TRACE_HIDE_INPUTS=true+export ATOMIC_AGENTS_TRACE_HIDE_OUTPUTS=true+export ATOMIC_AGENTS_TRACE_LLM_SPANS=falseopentelemetry-instrument python app.py```-The launcher discovers `atomic_agents` through the `opentelemetry_instrumentor` entry point and activates the-same adapter used by Phoenix and manual setup. Without the OpenInference dependencies, the loader skips the adapter.-`opentelemetry-bootstrap` does not automatically install Atomic Agents or its observability extra.+The launcher discovers `atomic_agents` through the `opentelemetry_instrumentor` entry point and activates the same+adapter as the manual setup. Without `opentelemetry-api` installed, the loader skips the adapter.+`opentelemetry-bootstrap` does not automatically install Atomic Agents or its extras.-To disable this adapter, set `OTEL_PYTHON_DISABLED_INSTRUMENTATIONS=atomic_agents` (or add `atomic_agents` to an-existing comma-separated list). Ordinary imports without a launcher or explicit setup remain uninstrumented.+To disable the adapter, set `OTEL_PYTHON_DISABLED_INSTRUMENTATIONS=atomic_agents` (or add `atomic_agents` to an+existing comma-separated list). Ordinary imports without a launcher or explicit setup stay uninstrumented.-Choose one provider configuration path: the launcher, Phoenix registration, or manual setup. Both discovery entry-points share one installation, so activating both does not duplicate Atomic Agents spans. The first provider/config-wins; a later call with different settings warns and keeps the existing installation. OpenTelemetry also keeps its-first global provider. Calling Phoenix registration after launcher startup therefore does not switch the export-destination. To combine explicit activations, configure a shared provider, or uninstrument before changing the-Atomic Agents provider/config as described below.+Choose one provider configuration path: the launcher, Phoenix registration, or manual setup. The discovery entry points+share one installation, so activating several does not duplicate spans. OpenTelemetry itself keeps its first global+provider, so calling Phoenix registration after launcher startup does not switch where spans are exported. See+"Re-instrumentation" for what a second `instrument()` call does.-The launcher adds AGENT and TOOL spans, with no parent CHAIN or provider LLM spans of its own. Install exactly one-instrumentor for your LLM provider and activate it through its supported setup. Use the grouping example below when-you want several agent/tool calls in one trace.+The launcher adds AGENT, TOOL and LLM-attempt spans and no parent span. The example above disables Atomic Agents' LLM attempt+spans for applications that also use a provider instrumentor. Leave `ATOMIC_AGENTS_TRACE_LLM_SPANS` unset to keep+attempt spans enabled. Use the grouping example below when you want several agent and tool calls in one trace.-## Setup with any OpenTelemetry backend (manual)+## Setup with Phoenix (automatic)++`register(auto_instrument=True)` also activates the OpenAI instrumentor, so set+`ATOMIC_AGENTS_TRACE_LLM_SPANS=false` (before import) to avoid two LLM layers:```python-from opentelemetry.sdk.trace import TracerProvider-from opentelemetry.sdk.trace.export import BatchSpanProcessor-from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter-from openinference.instrumentation import OITracer, TraceConfig-from openinference.instrumentation.openai import OpenAIInstrumentor+import os-from atomic_agents.observability import AtomicAgentsInstrumentor+os.environ.setdefault("ATOMIC_AGENTS_TRACE_LLM_SPANS", "false")-tracer_provider = TracerProvider()-tracer_provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter("http://localhost:6006/v1/traces")))-trace_config = TraceConfig()+from phoenix.otel import register-AtomicAgentsInstrumentor().instrument(tracer_provider=tracer_provider, config=trace_config)-OpenAIInstrumentor().instrument(tracer_provider=tracer_provider, config=trace_config)-tracer = OITracer(tracer_provider.get_tracer(__name__), config=trace_config)+tracer_provider = register(project_name="my-app", auto_instrument=True)```-The last line matters for the grouping section below: a plain SDK provider hands out a plain OpenTelemetry tracer,-and the `openinference_span_kind` argument, `set_input` and `set_output` only exist on an OpenInference `OITracer`.-Phoenix's `register()` returns a provider whose tracers already are `OITracer`s, so the automatic path skips this-step. Pass the same `TraceConfig` to every layer; the config given to `instrument()` configures only the Atomic Agents-spans.+`register(auto_instrument=True)` discovers the Atomic Agents instrumentor through the `openinference_instrumentor` entry+point and activates it together with every other OpenInference instrumentor in the environment. Install+`atomic-agents[phoenix]` (or `arize-phoenix-otel`, which brings the same packages) and do this once, at import time or in+your startup hook, before any agent runs. Phoenix reads `PHOENIX_COLLECTOR_ENDPOINT`, `PHOENIX_PROJECT` (alias+`PHOENIX_PROJECT_NAME`) and `PHOENIX_API_KEY` from the environment. With no endpoint set, `register()` exports over gRPC+to `localhost:4317`; set `PHOENIX_COLLECTOR_ENDPOINT=http://localhost:6006` to send HTTP to the port the Phoenix UI+serves on.-`instrument()` is idempotent. Calling it again with a different provider or config logs a warning and keeps the first-installation; call `uninstrument()` first to change it. `uninstrument()` restores the original methods on-`AtomicAgent` and on every tool class.+Two things to know about the automatic path:-## Grouping agent and tool calls into one trace+- Phoenix imports `atomic_agents` to load the entry point. That import is the same as any `import atomic_agents` in+ your application, including the Instructor streaming patch the framework applies at import.+- Existing `auto_instrument=True` users start exporting agent and tool payloads when they upgrade to a version with the+ instrumentor. Use the privacy controls below if that is not wanted.-Atomic Agents has no tool-calling loop: your code calls `agent.run(...)`, then `tool.run(...)`, then another agent.-Each of those is a root span, so without a parent they become separate traces. Open one parent span per unit of work:+Langfuse and other OpenInference-aware backends use the same OpenInference attributes, so the Phoenix-style setup, or+the manual setup with the `phoenix` extra installed, works for them too.++## What you get++| Call | Span name | Kind |+|------|-----------|------|+| `AtomicAgent.run`, `run_async`, `run_stream`, `run_async_stream` | `invoke_agent {agent name}` | AGENT |+| `run`, `run_async`, `arun` on any `BaseTool` subclass | `execute_tool {tool name}` | TOOL |+| Each model call Instructor makes during an agent run | `chat {request model}` | LLM attempt |++The agent name is `AgentConfig.name`, or the class name when you do not set one. The tool name is the name the model+sees (`tool_name` on the tool). Spans attach to the current context and never start a new trace when a parent span+exists, so your own request or job span becomes their parent. An agent span is the parent of its LLM attempt spans, and+a tool span nests under whatever span is current when the tool runs. A span that fails gets+status `ERROR`, the exception recorded, and `error.type` set to the exception class name.++Attributes, GenAI keys first. All of them are written whether or not the OpenInference packages are installed.++| Key | On | Value |+|-----|----|-------|+| `gen_ai.operation.name` | all | `invoke_agent`, `execute_tool` or `chat` |+| `gen_ai.agent.name` | AGENT | Agent name |+| `gen_ai.tool.name` | TOOL | Tool name the model sees |+| `gen_ai.tool.description` | TOOL | Tool description |+| `gen_ai.tool.type` | TOOL | `function` |+| `gen_ai.tool.call.arguments` | TOOL | Tool input as JSON (hidden by `hide_inputs`, size-capped) |+| `gen_ai.tool.call.result` | TOOL | Tool output as JSON (hidden by `hide_outputs`, size-capped) |+| `gen_ai.request.model` | AGENT, LLM | Model the agent was configured with, or the model in the completion call |+| `gen_ai.provider.name` | AGENT, LLM | Instructor provider mapped to the OpenTelemetry value, for example `azure.ai.openai`, `gcp.vertex_ai`, `aws.bedrock`; `openai` and `anthropic` pass through |+| `gen_ai.response.model` | AGENT, LLM | Model the provider reported; on the AGENT span, the last response's model |+| `gen_ai.usage.input_tokens` | AGENT, LLM | Input tokens (AGENT: total across attempts) |+| `gen_ai.usage.output_tokens` | AGENT, LLM | Output tokens |+| `gen_ai.usage.cache_read.input_tokens` | AGENT, LLM | Cached input tokens, when the provider reports them |+| `gen_ai.usage.reasoning.output_tokens` | AGENT, LLM | Reasoning tokens, when the provider reports them |+| `gen_ai.conversation.id` | all | Set by `using_conversation` |+| `error.type` | failed spans | Exception class name |++Atomic Agents' own keys:++| Key | On | Value |+|-----|----|-------|+| `atomic_agents.agent.class` | AGENT | For example `AtomicAgent[InputSchema, OutputSchema]` (a subclass reports its own class) |+| `atomic_agents.tool.class` | TOOL | Tool class name |+| `atomic_agents.model` | AGENT | Same as the request model |+| `atomic_agents.mode` | AGENT | Instructor mode |+| `atomic_agents.input_schema`, `atomic_agents.output_schema` | AGENT | Schema class names |+| `atomic_agents.history.message_count` | AGENT | History size entering the turn |+| `atomic_agents.context_providers` | AGENT | Registered context provider names |+| `atomic_agents.system_prompt.sha256` | AGENT | Hash of the static prompt sections (background, steps, output instructions). Context provider output is not included, so it identifies the prompt version even when inputs are hidden |+| `atomic_agents.input`, `atomic_agents.output` | AGENT | Agent input and output as JSON (hidden and size-capped as described under "Privacy and size"). A stream records the final response, or the last partial one |+| `atomic_agents.stream.closed_early` | AGENT | `true` when a stream consumer stopped early or the task was cancelled (not reported as an error) |+| `atomic_agents.llm.attempts` | AGENT | Number of model calls in the run |+| `atomic_agents.llm.last_validation_error` | AGENT | Text of the last validation error |+| `atomic_agents.llm.attempt` | LLM | 1-based attempt number |++OpenInference keys, optional. These are written only when the OpenInference packages are installed (the `phoenix`+extra), or when you force them with `TracingConfig(openinference_attributes=True)`; set it to `False` to drop them even+when the packages are present.++| Key | On | Value |+|-----|----|-------|+| `openinference.span.kind` | all | `AGENT`, `TOOL` or `LLM` |+| `input.value`, `input.mime_type` | AGENT, TOOL | Same payload as `atomic_agents.input` or `gen_ai.tool.call.arguments` |+| `output.value`, `output.mime_type` | AGENT, TOOL | Same payload as `atomic_agents.output` or `gen_ai.tool.call.result` |+| `agent.name` | AGENT | Agent name |+| `tool.name`, `tool.description` | TOOL | Same as the GenAI tool keys |+| `tool.parameters` | TOOL | The input JSON schema (size-capped, not hidden) |+| `session.id` | all | The conversation id, plus anything from `using_session`, `using_user` and `using_metadata` |++## LLM attempt spans++Instructor drives the model call and its retry loop, so the detail of each attempt only exists at the Atomic Agents+layer. The instrumentor registers hooks on the agent's Instructor client (once per client, on the first instrumented+run) and removes them again in `uninstrument()`:++- `completion:kwargs` starts the `chat {model}` span as a child of the agent span and counts the attempt.+- `completion:response` sets `gen_ai.response.model` and the token usage on that span and ends it with status OK.+- `completion:error` records the exception, sets `error.type` and ends the span with status `ERROR`.+- `parse:error` adds a `validation_error` event (with `atomic_agents.llm.attempt`) to the AGENT span and sets+ `atomic_agents.llm.last_validation_error`. Instructor then retries, which opens the next attempt span.+- `completion:usage` writes the running totals to the AGENT span: the four `gen_ai.usage.*` keys and+ `atomic_agents.llm.attempts`.++An attempt is one call to the model. A retry is any attempt after the first, normally because the response failed+validation, so two attempt spans under one agent span means one retry. Hooks you registered on the same client yourself+keep running next to these, because Instructor keeps a list of handlers per event. If the agent span ends while an+attempt is still open, the attempt span is ended with an unset status. The hooks never raise into your code.++OpenTelemetry's GenAI guidance says an agent framework should not emit model-call spans by default when the LLM library+already does. Atomic Agents emits them by default because the attempt and validation detail comes from Instructor, which+no provider instrumentor sees. If you also run a provider instrumentor (`openinference-instrumentation-openai` and+friends), turn the attempt spans off or you get two LLM layers:```python-with tracer.start_as_current_span("answer_question", openinference_span_kind="chain") as turn:- turn.set_input(question)- plan = planner.run(BasicChatInputSchema(chat_message=question))- calculation = calculator.run(CalculatorToolInputSchema(expression=plan.expression))- answer = answerer.run(CalculationResultSchema(question=question, expression=plan.expression, result=calculation.result))- turn.set_output(answer.chat_message)+AtomicAgentsInstrumentor().instrument(tracer_provider=tracer_provider, config=TracingConfig(llm_spans=False))+```++With `llm_spans=False` the usage totals and attempt count still land on the AGENT span, and the `validation_error` event+still fires.++## Naming and conversations++Give each agent a stable name, so spans group the same way across deployments and class renames:++```python+agent = AtomicAgent[InputSchema, OutputSchema](+ config=AgentConfig(client=client, model="gpt-5-mini", name="orchestrator_planning")+)```-`tracer` is the `OITracer` from the manual setup, or `tracer_provider.get_tracer(__name__)` on the Phoenix path. The-[phoenix-tracing example](https://github.com/eigenwise/atomic-agents/tree/main/atomic-examples/phoenix-tracing)-is exactly this flow, and the test suite runs the manual setup plus this grouping offline with an in-memory exporter.+The span becomes `invoke_agent orchestrator_planning`. Without a name, the class name is used.-## Sessions, users and metadata+Group related runs into a conversation with `using_conversation`:```python-from openinference.instrumentation import using_session, using_user, using_metadata+from atomic_agents.observability import using_conversation-with using_session(session_id=conversation_id), using_user(user_id=user.id), using_metadata({"tenant": tenant}):+with using_conversation(conversation_id):response = agent.run(user_message)```-Phoenix groups spans that share a `session.id` into a conversation view. For a streaming endpoint, put the context-manager inside the generator that yields the chunks, not around the endpoint function; the generator runs after the-endpoint returned, so a context manager around the endpoint body no longer applies.+Every AGENT, TOOL and LLM attempt span started inside the block gets `gen_ai.conversation.id` (and `session.id` when+OpenInference keys are on). With the `phoenix` extra installed, `using_session`, `using_user` and `using_metadata` from+`openinference.instrumentation` keep working, and a `using_session` value is copied to `gen_ai.conversation.id` when you+have not set one with `using_conversation`.++For a streaming endpoint, put the context manager inside the generator that yields the chunks, not around the endpoint+function. The generator runs after the endpoint returned, so a context manager around the endpoint body no longer+applies.++## Privacy and size++By default, agent and tool payloads are exported. `TracingConfig` controls that:++```python+TracingConfig(hide_inputs=True, hide_outputs=True, max_attribute_length=2048)+```++| Field | Default | Effect |+|-------|---------|--------|+| `hide_inputs` | `False` | Input payload values become `__REDACTED__` |+| `hide_outputs` | `False` | Output payload values become `__REDACTED__` |+| `max_attribute_length` | `4096` | Longer values are cut to this many characters and end in `...[truncated]` |+| `openinference_attributes` | auto | See "What you get" |+| `llm_spans` | `True` | See "LLM attempt spans" |++When no config is passed to `instrument()`, the settings come from the environment. `ATOMIC_AGENTS_TRACE_HIDE_INPUTS`+and `ATOMIC_AGENTS_TRACE_HIDE_OUTPUTS` are read first, then `OPENINFERENCE_HIDE_INPUTS` and `OPENINFERENCE_HIDE_OUTPUTS`+as a fallback (`true` or `1`). `ATOMIC_AGENTS_TRACE_MAX_ATTRIBUTE_LENGTH` sets the size cap.+`ATOMIC_AGENTS_TRACE_LLM_SPANS` disables attempt spans for `false` or `0` (case-insensitive); any other value or unset+keeps them enabled. If you pass your own+`TracingConfig(...)`, the environment is not read; use `TracingConfig.from_environment()` to start from it. An+OpenInference `TraceConfig` is also accepted for `config`; its `hide_inputs` and `hide_outputs` override those+environment settings, while the other settings still come from the environment.++A truncated value gets a companion attribute with the original size, for example `atomic_agents.input.original_length`.+Hiding wins over truncation, and a hidden payload is never serialized in the first place.++Covered by hiding and truncation: `atomic_agents.input`, `atomic_agents.output`, `input.value`, `output.value`,+`gen_ai.tool.call.arguments`, `gen_ai.tool.call.result` and `atomic_agents.llm.last_validation_error`. `tool.parameters`+is truncated but never hidden. Status descriptions and recorded exception messages are truncated too, and under+`hide_outputs` a Pydantic validation error message is replaced by `__REDACTED__` plus the error count, because the raw+model output is quoted inside it.++Instructor's own logging is separate from Atomic Agents' tracing: in the installed Instructor 1.17.0 source,+`instructor.v2.retry` (re-exported through `instructor.core.retry`) logs per-attempt validation failures at DEBUG+and retry exhaustion's raw last error, including model output, at ERROR regardless of `hide_outputs`.+Setting its level to ERROR does not silence exhaustion. To keep model output out of logs, use CRITICAL below+or a filter/handler that drops those records. The logger name and levels may change in later Instructor versions.++```python+import logging+logging.getLogger("instructor.v2.retry").setLevel(logging.CRITICAL)+```++Not covered: tool names and descriptions, the system prompt hash, and whatever you put in `using_metadata`. A provider+instrumentor and your own tracer apply their own settings, so configure them the same way if you want matching masking.+Multimodal inputs (images, audio, PDFs) are serialized into the input payload in full, so `max_attribute_length` is the+control for those.++## Grouping agent and tool calls into one trace++Atomic Agents has no tool-calling loop: your code calls `agent.run(...)`, then `tool.run(...)`, then another agent.+Each of those is a root span, so without a parent they become separate traces. Open one parent span per unit of work+with a plain OpenTelemetry tracer. It also answers "where do the seconds go": add child spans for retrieval or database+work next to the agent and tool calls.++```python+tracer = tracer_provider.get_tracer(__name__)++with tracer.start_as_current_span("answer_question"):+ with tracer.start_as_current_span("retrieve_documents"):+ documents = search_index(question)+ plan = planner.run(BasicChatInputSchema(chat_message=question))+ calculation = calculator.run(CalculatorToolInputSchema(expression=plan.expression))+ answer = answerer.run(CalculationResultSchema(question=question, expression=plan.expression, result=calculation.result))+```++The agent, tool and attempt spans become children of `answer_question`, and the slowest step stands out in the+waterfall. The [phoenix-tracing example](https://github.com/eigenwise/atomic-agents/tree/main/atomic-examples/phoenix-tracing)+is this flow. On the Phoenix path the provider's tracers are OpenInference `OITracer`s, which also take+`openinference_span_kind="chain"` and `set_input`/`set_output` if you want Phoenix to label the parent as a CHAIN.-## Privacy+## Re-instrumentation-Agent and tool payloads are exported as `input.value` / `output.value`. To redact them:+`instrument()` with the same provider and config as the active installation does nothing. With a different provider or+config it replaces them in place and logs a WARNING that says so. Spans already in flight finish on the provider they+started with; new spans use the new one. `uninstrument()` followed by `instrument()` is the clean way to change+settings, and `uninstrument()` restores the original methods on `AtomicAgent` and on every tool class.-- On the automatic Phoenix or OpenTelemetry launcher path, set `OPENINFERENCE_HIDE_INPUTS=true` and/or- `OPENINFERENCE_HIDE_OUTPUTS=true`. `TraceConfig()` reads these variables before the first installation.-- On the manual path, pass `TraceConfig(hide_inputs=True, hide_outputs=True)` to `instrument()`.+In a test suite, call `instrument(tracer_provider=provider)` with a fresh in-memory provider in each test. The replace+behavior means you do not need to uninstrument between tests, and the warning is expected there:-Redacted values are exported as `__REDACTED__`. Masking covers the `input.value` and `output.value` attributes of-Atomic Agents spans and nothing else: exception events and status descriptions (a validation error can quote input-values), tool names and descriptions, and whatever you put in `using_metadata` are exported as they are. The LLM-instrumentor and your own `OITracer` each take their own `TraceConfig`, so pass the same one to all three if you want-the same masking everywhere. `atomic_agents.system_prompt.sha256` stays available either way. It-hashes the static sections of a `SystemPromptGenerator` (background, steps, output instructions) and ignores what-context providers add per call, so it identifies the prompt version even when inputs are hidden. Multimodal inputs (images, audio, PDFs) are serialized into-`input.value` in full; cap attribute size with `OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT` if that is too much.+```python+from opentelemetry.sdk.trace import TracerProvider+from opentelemetry.sdk.trace.export import SimpleSpanProcessor+from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter++exporter = InMemorySpanExporter()+provider = TracerProvider()+provider.add_span_processor(SimpleSpanProcessor(exporter))+AtomicAgentsInstrumentor().instrument(tracer_provider=provider)+agent.run(user_input)+assert [span.name for span in exporter.get_finished_spans()][-1] == "invoke_agent my_agent"+```## Services and async-- Call `register(...)` (or `instrument(...)`) at import or in the startup hook so every worker process is instrumented.-- Use `register(batch=True)` (or a `BatchSpanProcessor`) in services so span export does not block request handling.-- Concurrent `run_async` calls, including `asyncio.gather` over several agents, get independent spans.+- Call `instrument(...)` (or `register(...)`) at import or in the startup hook so every worker process is instrumented.+- Use a `BatchSpanProcessor` (or `register(batch=True)`) in services so span export does not block request handling.+- Concurrent `run_async` calls, including `asyncio.gather` over several agents sharing one client, get independent+ spans and their own attempt spans.- Streaming token counts need `model_api_parameters={"stream_options": {"include_usage": True}}` for OpenAI.## Limitations@@ -191,12 +449,15 @@ context providers add per call, so it identifies the prompt version even when in- Methods bound before instrumentation stay untraced: `callback = agent.run` taken before `instrument()` keepscalling the original.- An `AtomicAgent` subclass that overrides `run` (or a stream method) without calling `super()` bypasses the wrapper- and emits no span; add `@tracer.agent` to such overrides yourself.-- Tool methods that are async generators are left untraced.+ and emits no span. Open a span in the override yourself, with `tracer.start_as_current_span(...)` or `@tracer.agent`+ on an `OITracer`.+- Tool methods that are async generators are left untraced. Open a span inside the generator body with+ `tracer.start_as_current_span(...)`, or use `@tracer.tool` on an `OITracer`.- A plain `ThreadPoolExecutor` starts its worker with an empty context, so a tool that submits work to one loses thecurrent span and the re-entrancy guard inside the pool. Submit through a copied context instead, as the forge toolsdo: `executor.submit(contextvars.copy_context().run, asyncio.run, self.run_async(params))`. `asyncio.to_thread` and`asyncio.run` already copy the context.- The AGENT span records the final (or last partial) response of a stream, not every chunk.-- The system prompt text is not on the AGENT span; it is the first message on the LLM span. The AGENT span carries its- SHA-256 instead.+- The system prompt text is not on any span. The AGENT span carries its SHA-256 instead.+- Datadog and PostHog read prompt and response text from `gen_ai.input.messages` and `gen_ai.output.messages`, which+ are not emitted.diff --git a/pyproject.toml b/pyproject.tomlindex 87e3464..e720bc5 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -37,10 +37,11 @@ dependencies = [][project.optional-dependencies]-observability = [+observability = ["opentelemetry-api>=1.30.0,<2.0.0"]+phoenix = [+ "opentelemetry-api>=1.30.0,<2.0.0","openinference-instrumentation>=0.1.38,<0.2.0","openinference-semantic-conventions>=0.1.20,<0.2.0",- "opentelemetry-api>=1.30.0,<2.0.0",][project.urls]diff --git a/uv.lock b/uv.lockindex aa3aaa5..8ced42d 100644--- a/uv.lock+++ b/uv.lock@@ -284,6 +284,9 @@ dependencies = [[package.optional-dependencies]observability = [+ { name = "opentelemetry-api" },+]+phoenix = [{ name = "openinference-instrumentation" },{ name = "openinference-semantic-conventions" },{ name = "opentelemetry-api" },@@ -320,9 +323,10 @@ requires-dist = [{ name = "instructor", specifier = ">=1.17.0,<2" },{ name = "litellm", specifier = ">=1.50.0,<2.0.0" },{ name = "mcp", extras = ["cli"], specifier = ">=1.8.0,<2" },- { name = "openinference-instrumentation", marker = "extra == 'observability'", specifier = ">=0.1.38,<0.2.0" },- { name = "openinference-semantic-conventions", marker = "extra == 'observability'", specifier = ">=0.1.20,<0.2.0" },+ { name = "openinference-instrumentation", marker = "extra == 'phoenix'", specifier = ">=0.1.38,<0.2.0" },+ { name = "openinference-semantic-conventions", marker = "extra == 'phoenix'", specifier = ">=0.1.20,<0.2.0" },{ name = "opentelemetry-api", marker = "extra == 'observability'", specifier = ">=1.30.0,<2.0.0" },+ { name = "opentelemetry-api", marker = "extra == 'phoenix'", specifier = ">=1.30.0,<2.0.0" },{ name = "pydantic", specifier = ">=2.11.4,<3.0.0" },{ name = "pyfiglet", specifier = ">=1.0.2,<2.0.0" },{ name = "pyyaml", specifier = ">=6.0.2,<7.0.0" },@@ -330,7 +334,7 @@ requires-dist = [{ name = "rich", specifier = ">=13.7.1,<14.0.0" },{ name = "textual", specifier = ">=5.3.0,<6.0.0" },]-provides-extras = ["observability"]+provides-extras = ["observability", "phoenix"][package.metadata.requires-dev]dev = [@@ -3161,7 +3165,7 @@ version = "1.0.0"source = { editable = "atomic-examples/phoenix-tracing" }dependencies = [{ name = "arize-phoenix-otel" },- { name = "atomic-agents", extra = ["observability"] },+ { name = "atomic-agents", extra = ["phoenix"] },{ name = "instructor" },{ name = "openai" },{ name = "openinference-instrumentation-openai" },@@ -3172,7 +3176,7 @@ dependencies = [[package.metadata]requires-dist = [{ name = "arize-phoenix-otel", specifier = ">=0.17.0,<1.0.0" },- { name = "atomic-agents", extras = ["observability"], editable = "." },+ { name = "atomic-agents", extras = ["phoenix"], editable = "." },{ name = "instructor", specifier = ">=1.17.0,<2" },{ name = "openai", specifier = ">=2.0.0,<3.0.0" },{ name = "openinference-instrumentation-openai", specifier = ">=0.1.30,<0.2.0" },