JiuwenSwarm is an intelligent AI Agent built on openJiuwen. It extends the powerful capabilities of large language models directly to your fingertips through various communication apps you use daily.
Python
from __future__ import annotations
import ast
import asyncio
from collections.abc import AsyncIterator, Iterator
import inspect
from types import SimpleNamespace
import threading
import textwrap
import time
import pytest
from opentelemetry import context as otel_context
from opentelemetry import trace
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import InMemoryMetricReader
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.sampling import ALWAYS_OFF
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
InMemorySpanExporter,
)
from openjiuwen.agent_teams.observability import (
ObservabilityConfig,
init_observability,
shutdown_observability,
)
from openjiuwen.agent_teams.observability.span_context import (
clear_team_span,
set_current_agent_span,
set_team_span,
)
from openjiuwen.core.runner import Runner
from openjiuwen.core.foundation.tool.base import Tool, ToolCard
from openjiuwen.harness.tools.base_tool import ToolOutput
from openjiuwen.core.runner.callback import (
AgentEvents,
AsyncCallbackFramework,
LLMCallEvents,
ToolCallEvents,
)
from jiuwenswarm.telemetry.config import TelemetryConfig
from jiuwenswarm.extensions.identity_provider import IdentityInfo, IdentityStore
import jiuwenswarm.telemetry.enrichment.callbacks as callbacks_module
from jiuwenswarm.telemetry.enrichment.callbacks import (
EnrichmentStateRegistry,
INPUT_PRIORITY,
NAMESPACE,
OUTPUT_PRIORITY,
MetricCallStateRegistry,
RichTelemetryCallbacks,
SkillSessionRegistry,
)
from jiuwenswarm.telemetry.enrichment.tokens import ContextTokenBreakdown
from jiuwenswarm.telemetry.span_registry import SpanRegistryProcessor
from jiuwenswarm.telemetry.metrics import TelemetryMetrics, metrics_channel_id
from jiuwenswarm.telemetry.metrics import metrics_adapter_mode_id, metrics_mode_id
def test_callback_label_keeps_searching_after_string_conversion_failure() -> None:
class Unstringable:
@staticmethod
def __str__() -> str:
raise ValueError("cannot stringify")
assert (
RichTelemetryCallbacks._callback_label(
{"provider": Unstringable()},
{"provider": "fallback"},
"provider",
)
== "fallback"
)
def test_callback_label_does_not_continue_directly_from_exception_handler() -> None:
source = textwrap.dedent(
inspect.getsource(RichTelemetryCallbacks._callback_label)
)
tree = ast.parse(source)
assert not any(
isinstance(child, ast.Continue)
for handler in ast.walk(tree)
if isinstance(handler, ast.ExceptHandler)
for child in ast.walk(handler)
)
def test_validate_core_ordering_is_static() -> None:
descriptor = RichTelemetryCallbacks.__dict__["validate_core_ordering"]
assert isinstance(descriptor, staticmethod)
def _metric_points(reader: InMemoryMetricReader, name: str) -> list:
data = reader.get_metrics_data()
if data is None:
return []
return [
point
for resource_metrics in data.resource_metrics
for scope_metrics in resource_metrics.scope_metrics
for metric in scope_metrics.metrics
if metric.name == name
for point in metric.data.data_points
]
class _Metrics:
def add(self, name, value, attributes=None):
del name, value, attributes
class _RecordingMetrics:
def __init__(self, error: BaseException | None = None) -> None:
self.calls: list[tuple[str, str, int | float, dict]] = []
self.error = error
def add(self, name, value, attributes=None):
if self.error is not None:
raise self.error
self.calls.append(("add", name, value, dict(attributes or {})))
def record(self, name, value, attributes=None):
if self.error is not None:
raise self.error
self.calls.append(("record", name, value, dict(attributes or {})))
class _RealSkillLifecycleTool(Tool):
def __init__(
self,
name: str,
*,
error: BaseException | None = None,
success: bool = True,
) -> None:
super().__init__(ToolCard(name=name, description="Telemetry skill fixture"))
self._error = error
self._success = success
async def invoke(self, inputs, **kwargs):
del kwargs
if self._error is not None:
raise self._error
if not self._success:
return ToolOutput(success=False, error="skill load failed")
return ToolOutput(
success=True,
data={
"skill_directory": f"/skills/{inputs['skill_name']}",
"skill_content": "body",
},
)
async def stream(self, inputs, **kwargs):
del inputs, kwargs
if False:
yield None
class _FailingFramework(AsyncCallbackFramework):
def __init__(
self,
*,
fail_register_at: int | None = None,
fail_unregister_at: int | None = None,
) -> None:
super().__init__()
self.fail_register_at = fail_register_at
self.fail_unregister_at = fail_unregister_at
self.register_count = 0
self.unregister_count = 0
self.fail_unregister_callback = None
async def register(self, event, callback, **kwargs):
self.register_count += 1
if self.register_count == self.fail_register_at:
raise RuntimeError("register failed")
return await super().register(event, callback, **kwargs)
async def unregister(self, event, callback):
self.unregister_count += 1
if self.unregister_count == self.fail_unregister_at:
raise RuntimeError("rollback unregister failed")
if callback is self.fail_unregister_callback:
self.fail_unregister_callback = None
raise RuntimeError("unregister failed")
return await super().unregister(event, callback)
def _sync_core_redaction_with_swarm(swarm_callbacks: RichTelemetryCallbacks) -> None:
"""Point the core handler's redaction policy at the swarm TelemetryConfig.
Production maps ``TelemetryConfig.redact_prompts``/``redact_completions``
1:1 into the core ``ObservabilityConfig`` (see TelemetryRuntime
``_start_agent_components``). The fixture initializes the core runtime
with its default config before swarm tests swap ``callbacks._config``
mid-test, so the core ``OtelCallbackHandler`` would keep writing plaintext
``gen_ai.*.messages`` — a state production cannot produce. A live proxy
keeps both sides on the same policy, mirroring the production mapping.
"""
from openjiuwen.extensions.observability.setup import get_observability_runtime
handler = get_observability_runtime()._callback_handler
if handler is None:
return
core_config = handler._config
class _SwarmSyncedRedactionConfig:
def __getattr__(self, name: str):
if name in (
"redact_prompts",
"redact_completions",
"attribute_value_max_length",
):
return getattr(swarm_callbacks._config, name)
return getattr(core_config, name)
handler._config = _SwarmSyncedRedactionConfig()
# The core handler writes ``gen_ai.input.messages``/``gen_ai.output.messages``
# unconditionally, duplicating (on output events, after) the swarm-authored
# attributes. Swarm's enterprise shape is the authoritative surface and the
# only writer that honors ``log_messages``/serializer-failure isolation, so
# the fixture suppresses the core duplicate writers — the pre-rewrite core
# behavior these tests encode.
handler._record_structured_output = lambda span, response, **kwargs: None
handler._record_standard_structured_input = lambda span, messages: None
@pytest.fixture
async def telemetry_env() -> AsyncIterator[SimpleNamespace]:
shutdown_observability()
exporter = InMemorySpanExporter()
provider = TracerProvider(
resource=Resource.create(
{
"service.name": "callback-test",
"service.version": "1.2.3",
"jiuwenclaw.claw.id": "claw-test",
}
)
)
registry = SpanRegistryProcessor(max_spans=64, ttl_seconds=60)
provider.add_span_processor(registry)
provider.add_span_processor(SimpleSpanProcessor(exporter))
reader = InMemoryMetricReader()
meter_provider = MeterProvider(metric_readers=[reader])
metrics = TelemetryMetrics(meter_provider)
callbacks = RichTelemetryCallbacks(
span_registry=registry,
metrics=metrics,
config=TelemetryConfig(enabled=True, claw_id="claw-config"),
)
init_observability(
ObservabilityConfig(enabled=True, backend="otlp"),
tracer_provider_override=provider,
owns_provider=False,
)
await callbacks.register(Runner.callback_framework)
_sync_core_redaction_with_swarm(callbacks)
parent = provider.get_tracer("test").start_span(
"agent.worker.invoke",
attributes={"gen_ai.request.model": "core-model"},
)
set_current_agent_span(parent)
set_team_span(parent)
context_token = otel_context.attach(trace.set_span_in_context(parent))
try:
yield SimpleNamespace(
callbacks=callbacks,
exporter=exporter,
framework=Runner.callback_framework,
meter_provider=meter_provider,
parent=parent,
provider=provider,
reader=reader,
registry=registry,
)
finally:
otel_context.detach(context_token)
set_current_agent_span(None)
clear_team_span()
await callbacks.unregister(Runner.callback_framework)
shutdown_observability()
if parent.is_recording():
parent.end()
provider.force_flush()
meter_provider.shutdown()
provider.shutdown()
def record(self, name, value, attributes=None):
del name, value, attributes
@pytest.fixture
async def unsampled_telemetry_env() -> AsyncIterator[SimpleNamespace]:
shutdown_observability()
exporter = InMemorySpanExporter()
provider = TracerProvider(sampler=ALWAYS_OFF)
registry = SpanRegistryProcessor(max_spans=64, ttl_seconds=60)
provider.add_span_processor(registry)
provider.add_span_processor(SimpleSpanProcessor(exporter))
reader = InMemoryMetricReader()
meter_provider = MeterProvider(metric_readers=[reader])
callbacks = RichTelemetryCallbacks(
span_registry=registry,
metrics=TelemetryMetrics(meter_provider),
config=TelemetryConfig(enabled=True),
)
init_observability(
ObservabilityConfig(enabled=True, backend="otlp"),
tracer_provider_override=provider,
owns_provider=False,
)
await callbacks.register(Runner.callback_framework)
_sync_core_redaction_with_swarm(callbacks)
parent = provider.get_tracer("test").start_span("agent.unsampled.invoke")
assert not parent.is_recording()
set_current_agent_span(parent)
set_team_span(parent)
context_token = otel_context.attach(trace.set_span_in_context(parent))
try:
yield SimpleNamespace(
callbacks=callbacks,
exporter=exporter,
framework=Runner.callback_framework,
meter_provider=meter_provider,
parent=parent,
provider=provider,
reader=reader,
registry=registry,
)
finally:
otel_context.detach(context_token)
set_current_agent_span(None)
clear_team_span()
await callbacks.unregister(Runner.callback_framework)
shutdown_observability()
parent.end()
meter_provider.shutdown()
provider.shutdown()
@pytest.fixture
def request_metric_channel() -> Iterator[None]:
token = metrics_channel_id.set("web")
try:
yield
finally:
metrics_channel_id.reset(token)
@pytest.fixture
def request_metric_modes() -> Iterator[None]:
mode_token = metrics_mode_id.set("agent")
adapter_token = metrics_adapter_mode_id.set("flash")
try:
yield
finally:
metrics_adapter_mode_id.reset(adapter_token)
metrics_mode_id.reset(mode_token)
def _decorate_like_real_model(
framework: AsyncCallbackFramework,
invoke,
*,
model_config: object,
model_client_config: object,
):
extra = {
"model_config": model_config,
"model_client_config": model_client_config,
}
wrapped = framework.emit_before(
LLMCallEvents.LLM_INVOKE_INPUT,
extra_kwargs=extra,
)(invoke)
wrapped = framework.transform_io(
input_event=LLMCallEvents.LLM_INVOKE_INPUT,
output_event=LLMCallEvents.LLM_INVOKE_OUTPUT,
)(wrapped)
return framework.emit_after(
LLMCallEvents.LLM_INVOKE_OUTPUT,
extra_kwargs=extra,
)(wrapped)
def _decorate_like_real_agent_stream(
framework: AsyncCallbackFramework,
stream,
):
wrapped = framework.emit_before(AgentEvents.AGENT_STREAM_INPUT)(stream)
wrapped = framework.transform_io(
input_event=AgentEvents.AGENT_STREAM_INPUT,
output_event=AgentEvents.AGENT_STREAM_OUTPUT,
)(wrapped)
return framework.emit_after(
AgentEvents.AGENT_STREAM_OUTPUT,
item_key="result",
)(wrapped)
@pytest.mark.asyncio
async def test_registers_complete_real_event_set_with_fixed_priorities() -> None:
framework = AsyncCallbackFramework()
callbacks = RichTelemetryCallbacks(
span_registry=SpanRegistryProcessor(max_spans=16, ttl_seconds=60),
metrics=_Metrics(),
config=TelemetryConfig(),
)
await callbacks.register(framework)
input_events = {
LLMCallEvents.LLM_INVOKE_INPUT,
LLMCallEvents.LLM_STREAM_INPUT,
ToolCallEvents.TOOL_CALL_STARTED,
AgentEvents.AGENT_INVOKE_INPUT,
AgentEvents.AGENT_STREAM_INPUT,
}
output_events = {
LLMCallEvents.LLM_STREAM_OUTPUT,
LLMCallEvents.LLM_INVOKE_OUTPUT,
LLMCallEvents.LLM_OUTPUT,
LLMCallEvents.LLM_CALL_ERROR,
ToolCallEvents.TOOL_CALL_FINISHED,
ToolCallEvents.TOOL_CALL_ERROR,
AgentEvents.AGENT_INVOKE_OUTPUT,
AgentEvents.AGENT_STREAM_OUTPUT,
}
assert len(callbacks._registered) == len(input_events | output_events)
for event in input_events:
assert framework.list_callbacks(event) == [
{
"name": framework.list_callbacks(event)[0]["name"],
"priority": INPUT_PRIORITY,
"enabled": True,
"namespace": NAMESPACE,
"tags": [],
"once": False,
"max_retries": 0,
"timeout": None,
}
]
for event in output_events:
assert framework.list_callbacks(event)[0]["priority"] == OUTPUT_PRIORITY
assert framework.list_callbacks(event)[0]["namespace"] == NAMESPACE
@pytest.mark.asyncio
async def test_real_llm_callbacks_enrich_the_single_core_span(
telemetry_env: SimpleNamespace,
) -> None:
result = SimpleNamespace(
content="world",
finish_reason="stop",
tool_calls=[],
usage_metadata=SimpleNamespace(
input_tokens=7,
output_tokens=3,
cache_read_input_tokens=2,
cache_creation_input_tokens=4,
reasoning_tokens=1,
),
)
await telemetry_env.framework.trigger(
LLMCallEvents.LLM_INVOKE_INPUT,
messages=[{"role": "user", "content": "hello"}],
model="enrichment-model",
temperature=0.2,
)
callback_results = await telemetry_env.framework.trigger(
LLMCallEvents.LLM_INVOKE_OUTPUT,
result=result,
)
telemetry_env.provider.force_flush()
spans = [
span
for span in telemetry_env.exporter.get_finished_spans()
if span.name == "llm.call"
]
assert len(spans) == 1
span = spans[0]
assert span.attributes["gen_ai.request.model"] == "enrichment-model"
assert span.attributes["gen_ai.input.messages.count"] == 1
assert span.attributes["gen_ai.input.messages.total_length"] == 5
assert span.attributes["gen_ai.decision.type"] == "answer"
assert span.attributes["gen_ai.usage.input_tokens"] == 7
assert span.attributes["gen_ai.usage.cache_creation.input_tokens"] == 4
assert span.attributes["gen_ai.response.finish_reasons"] == ("stop",)
assert span.attributes["gen_ai.response.finish_reason"] == "stop"
assert any(event.name == "gen_ai.user.message" for event in span.events)
assert callback_results[0] is result
@pytest.mark.asyncio
async def test_real_model_decorators_normalize_positional_input_and_llm_output_once(
telemetry_env: SimpleNamespace,
) -> None:
messages = [{"role": "user", "content": "positional prompt"}]
tool_calls = [
{
"id": "call-weather",
"function": {
"name": "weather",
"arguments": '{"city":"Paris"}',
},
}
]
usage = SimpleNamespace(
input_tokens=11,
output_tokens=5,
cache_read_input_tokens=3,
cache_creation_input_tokens=2,
reasoning_tokens=1,
)
business_result = SimpleNamespace(
content="parsed business result",
reasoning_content="parsed reasoning",
finish_reason="tool_calls",
tool_calls=tool_calls,
usage_metadata=usage,
)
state_counts_after_inner_output: list[tuple[int, int]] = []
async def invoke(received_messages, **kwargs):
assert received_messages is messages
assert kwargs["tools"][0]["function"]["name"] == "weather"
await telemetry_env.framework.trigger(
LLMCallEvents.LLM_OUTPUT,
response="observable answer",
reasoning_content="observable reasoning",
tool_calls=tool_calls,
usage=usage,
model_name="response-model",
model_provider="OpenAI",
)
state_counts_after_inner_output.append(
(
telemetry_env.callbacks._span_state.active_count(),
telemetry_env.callbacks._metric_state.active_count(),
)
)
return business_result
wrapped = _decorate_like_real_model(
telemetry_env.framework,
invoke,
model_config=SimpleNamespace(
model="configured-model",
temperature=0.25,
top_p=0.75,
),
model_client_config=SimpleNamespace(client_provider="OpenAI"),
)
returned = await wrapped(
messages,
tools=[{"function": {"name": "weather"}}],
)
telemetry_env.provider.force_flush()
assert returned is business_result
# AgentCore now treats LLM_OUTPUT as the terminal event for both streaming
# and non-streaming calls, so the JiuwenClaw state is settled before the
# core callback closes its owned span.
assert state_counts_after_inner_output == [(0, 0)]
spans = [
span
for span in telemetry_env.exporter.get_finished_spans()
if span.name == "llm.call"
]
assert len(spans) == 1
attrs = spans[0].attributes
assert attrs["gen_ai.input.messages.count"] == 1
assert "positional prompt" in attrs["gen_ai.input.messages"]
assert attrs["gen_ai.request.model"] == "configured-model"
assert attrs["gen_ai.request.temperature"] == 0.25
assert attrs["gen_ai.request.top_p"] == 0.75
assert attrs["gen_ai.response.model"] == "response-model"
assert attrs["gen_ai.span.type"] == "model"
assert attrs["gen_ai.decision.type"] == "tool_call"
assert attrs["gen_ai.decision.tool_names"] == ("weather",)
assert "observable answer" in attrs["gen_ai.output.messages"]
assert "observable reasoning" in attrs["gen_ai.output.messages"]
assert "call-weather" in attrs["gen_ai.output.messages"]
assert 'city\\":\\"Paris' in attrs["gen_ai.output.messages"]
assert attrs["gen_ai.usage.cache_read.input_tokens"] == 3
assert attrs["gen_ai.usage.cache_creation.input_tokens"] == 2
assert attrs["gen_ai.usage.cache_read_tokens"] == 3
assert attrs["gen_ai.usage.cache_creation_tokens"] == 2
assert telemetry_env.callbacks._span_state.active_count() == 0
assert telemetry_env.callbacks._metric_state.active_count() == 0
operation_counts = _metric_points(
telemetry_env.reader,
"gen_ai.client.operation.count",
)
assert len(operation_counts) == 1
assert operation_counts[0].value == 1
assistant_events = [
event for event in spans[0].events if event.name == "gen_ai.assistant.message"
]
assert len(assistant_events) == 1
assert "observable answer" in assistant_events[0].attributes["content"]
@pytest.mark.asyncio
async def test_real_tool_callbacks_enrich_the_single_core_span(
telemetry_env: SimpleNamespace,
) -> None:
result = {"answer": "sunny"}
await telemetry_env.framework.trigger(
ToolCallEvents.TOOL_CALL_STARTED,
tool_name="weather",
tool_id="call-1",
inputs=(("Paris",), {}),
)
callback_results = await telemetry_env.framework.trigger(
ToolCallEvents.TOOL_CALL_FINISHED,
tool_name="weather",
tool_id="call-1",
inputs=(("Paris",), {}),
result=result,
)
telemetry_env.provider.force_flush()
spans = [
span
for span in telemetry_env.exporter.get_finished_spans()
if span.name == "tool.weather"
]
assert len(spans) == 1
span = spans[0]
assert span.attributes["gen_ai.tool.name"] == "weather"
assert span.attributes["gen_ai.tool.call.id"] == "call-1"
assert span.attributes["gen_ai.span.type"] == "tool"
assert any(event.name == "tool.arguments" for event in span.events)
assert any(event.name == "tool.result" for event in span.events)
assert callback_results[0] == result
@pytest.mark.asyncio
async def test_streaming_tool_finish_without_result_does_not_invent_null_payload(
telemetry_env: SimpleNamespace,
) -> None:
await telemetry_env.framework.trigger(
ToolCallEvents.TOOL_CALL_STARTED,
tool_name="streaming_tool",
tool_id="stream-card-id",
inputs=(("input",), {}),
)
await telemetry_env.framework.trigger(
ToolCallEvents.TOOL_CALL_FINISHED,
tool_name="streaming_tool",
tool_id="stream-card-id",
)
telemetry_env.provider.force_flush()
span = [
item
for item in telemetry_env.exporter.get_finished_spans()
if item.name == "tool.streaming_tool"
][0]
assert "gen_ai.tool.result" not in span.attributes
assert not any(event.name == "tool.result" for event in span.events)
assert telemetry_env.callbacks._metric_state.active_count() == 0
assert telemetry_env.callbacks._span_state.active_count() == 0
@pytest.mark.asyncio
async def test_llm_metrics_state_and_streaming_ttft_are_finalized(
telemetry_env: SimpleNamespace,
) -> None:
await telemetry_env.framework.trigger(
LLMCallEvents.LLM_STREAM_INPUT,
messages=[{"role": "user", "content": "stream"}],
model="stream-model",
tools=[{"function": {"name": "weather", "description": "forecast"}}],
)
assert telemetry_env.callbacks._span_state.active_count() == 1
assert telemetry_env.callbacks._metric_state.active_count() == 1
from openjiuwen.agent_teams.observability.span_context import get_current_llm_span
span = get_current_llm_span()
assert span is not None
span.otel_llm_state.start_ns = time.monotonic_ns() - 20_000_000
chunk = SimpleNamespace(content="first", finish_reason=None)
results = await telemetry_env.framework.trigger(
LLMCallEvents.LLM_STREAM_OUTPUT,
result=chunk,
)
final_results = await telemetry_env.framework.trigger(
LLMCallEvents.LLM_OUTPUT,
response="done",
usage=SimpleNamespace(input_tokens=5, output_tokens=2),
)
telemetry_env.provider.force_flush()
assert results[0] is chunk
assert final_results[0] == "done"
assert telemetry_env.callbacks._span_state.active_count() == 0
assert telemetry_env.callbacks._metric_state.active_count() == 0
finished = [
item
for item in telemetry_env.exporter.get_finished_spans()
if item.name == "llm.call"
]
assert len(finished) == 1
assert finished[0].attributes["gen_ai.streaming.first_token"] is True
ttft_points = _metric_points(
telemetry_env.reader, "gen_ai.client.token.first_token_duration"
)
assert len(ttft_points) == 1
assert 0.01 <= ttft_points[0].sum <= 1.0
duration_points = _metric_points(
telemetry_env.reader, "gen_ai.client.operation.duration"
)
count_points = _metric_points(telemetry_env.reader, "gen_ai.client.operation.count")
assert len(duration_points) == 1
assert len(count_points) == 1
@pytest.mark.asyncio
@pytest.mark.parametrize("provider_field", ["client_provider", "provider"])
async def test_enterprise_metric_labels_llm_success_are_exact_and_provider_wins(
telemetry_env: SimpleNamespace,
provider_field: str,
) -> None:
telemetry_env.parent.set_attribute("jiuwenclaw.channel.id", "web")
telemetry_env.parent.set_attribute("jiuwenclaw.agent.name", "main-agent")
telemetry_env.parent.set_attribute("jiuwenclaw.mode", "agent")
telemetry_env.parent.set_attribute("jiuwenclaw.adapter.mode", "flash")
telemetry_env.parent.set_attribute("gen_ai.provider.name", "span-provider")
await telemetry_env.framework.trigger(
LLMCallEvents.LLM_INVOKE_INPUT,
messages=[{"role": "user", "content": "hello"}],
model="mock-model",
model_client_config={provider_field: "OpenAI"},
)
await telemetry_env.framework.trigger(
LLMCallEvents.LLM_INVOKE_OUTPUT,
result=SimpleNamespace(
content="ok",
tool_calls=[],
finish_reason="stop",
usage_metadata=SimpleNamespace(input_tokens=4, output_tokens=2),
),
)
duration_points = _metric_points(
telemetry_env.reader,
"gen_ai.client.operation.duration",
)
count_points = _metric_points(
telemetry_env.reader,
"gen_ai.client.operation.count",
)
token_points = _metric_points(
telemetry_env.reader,
"gen_ai.client.token.usage",
)
assert len(duration_points) == 1
assert len(count_points) == 1
assert len(token_points) == 2
assert dict(duration_points[0].attributes) == {
"gen_ai.request.model": "mock-model",
"gen_ai.system": "openai",
"jiuwenclaw.channel.id": "web",
"jiuwenclaw.mode": "agent",
"jiuwenclaw.adapter.mode": "flash",
}
assert dict(count_points[0].attributes) == {
"gen_ai.request.model": "mock-model",
"status": "success",
"jiuwenclaw.channel.id": "web",
"jiuwenclaw.mode": "agent",
"jiuwenclaw.adapter.mode": "flash",
}
assert {
point.attributes["gen_ai.token.type"]: dict(point.attributes)
for point in token_points
} == {
"input": {
"gen_ai.request.model": "mock-model",
"gen_ai.system": "openai",
"jiuwenclaw.channel.id": "web",
"jiuwenclaw.mode": "agent",
"jiuwenclaw.adapter.mode": "flash",
"gen_ai.token.type": "input",
},
"output": {
"gen_ai.request.model": "mock-model",
"gen_ai.system": "openai",
"jiuwenclaw.channel.id": "web",
"jiuwenclaw.mode": "agent",
"jiuwenclaw.adapter.mode": "flash",
"gen_ai.token.type": "output",
},
}
@pytest.mark.asyncio
async def test_enterprise_metric_labels_llm_error_uses_current_core_span_provider(
telemetry_env: SimpleNamespace,
) -> None:
telemetry_env.parent.set_attribute("jiuwenclaw.channel.id", "cli")
await telemetry_env.framework.trigger(
LLMCallEvents.LLM_INVOKE_INPUT,
messages=[{"role": "user", "content": "fail"}],
model="error-model",
)
await telemetry_env.framework.trigger(
LLMCallEvents.LLM_CALL_ERROR,
error=RuntimeError("model failed"),
)
duration_points = _metric_points(
telemetry_env.reader,
"gen_ai.client.operation.duration",
)
count_points = _metric_points(
telemetry_env.reader,
"gen_ai.client.operation.count",
)
assert len(duration_points) == 1
assert len(count_points) == 1
assert dict(duration_points[0].attributes) == {
"gen_ai.request.model": "error-model",
"gen_ai.system": "openjiuwen-agent-teams",
"jiuwenclaw.channel.id": "cli",
}
assert dict(count_points[0].attributes) == {
"gen_ai.request.model": "error-model",
"status": "error",
"jiuwenclaw.channel.id": "cli",
}
@pytest.mark.asyncio
async def test_enterprise_metric_labels_tool_error_are_exact_without_call_id(
telemetry_env: SimpleNamespace,
) -> None:
telemetry_env.parent.set_attribute("jiuwenclaw.channel.id", "web")
telemetry_env.parent.set_attribute("jiuwenclaw.mode", "agent")
telemetry_env.parent.set_attribute("jiuwenclaw.adapter.mode", "flash")
await telemetry_env.framework.trigger(
ToolCallEvents.TOOL_CALL_STARTED,
tool_name="weather",
tool_id="call-1",
inputs={},
)
await telemetry_env.framework.trigger(
ToolCallEvents.TOOL_CALL_ERROR,
tool_name="weather",
tool_id="call-1",
error=RuntimeError("tool failed"),
)
telemetry_env.provider.force_flush()
expected = {
"gen_ai.tool.name": "weather",
"jiuwenclaw.channel.id": "web",
"jiuwenclaw.mode": "agent",
"jiuwenclaw.adapter.mode": "flash",
}
for metric_name in (
"gen_ai.tool.duration",
"gen_ai.tool.call.count",
"gen_ai.tool.error.count",
):
points = _metric_points(telemetry_env.reader, metric_name)
assert len(points) == 1
assert dict(points[0].attributes) == expected
span = [
item
for item in telemetry_env.exporter.get_finished_spans()
if item.name == "tool.weather"
][0]
assert span.attributes["gen_ai.tool.call.id"] == "call-1"
@pytest.mark.asyncio
async def test_enterprise_metric_labels_filter_empty_values_and_provider_is_unknown() -> None:
reader = InMemoryMetricReader()
meter_provider = MeterProvider(metric_readers=[reader])
callbacks = RichTelemetryCallbacks(
span_registry=SpanRegistryProcessor(max_spans=8, ttl_seconds=60),
metrics=TelemetryMetrics(meter_provider),
config=TelemetryConfig(enabled=True, service_name="must-not-be-provider"),
)
framework = AsyncCallbackFramework()
await callbacks.register(framework)
try:
await framework.trigger(
LLMCallEvents.LLM_INVOKE_INPUT,
messages=[],
model="",
)
await framework.trigger(
LLMCallEvents.LLM_INVOKE_OUTPUT,
result=SimpleNamespace(content="ok", tool_calls=[]),
)
await framework.trigger(AgentEvents.AGENT_INVOKE_INPUT, {})
await framework.trigger(AgentEvents.AGENT_INVOKE_OUTPUT, result="done")
await framework.trigger(
ToolCallEvents.TOOL_CALL_STARTED,
tool_name="",
tool_id="",
inputs={},
)
await framework.trigger(
ToolCallEvents.TOOL_CALL_FINISHED,
tool_name="",
tool_id="",
result={"ok": True},
)
llm_duration = _metric_points(
reader,
"gen_ai.client.operation.duration",
)
llm_count = _metric_points(reader, "gen_ai.client.operation.count")
agent_duration = _metric_points(reader, "jiuwenclaw.agent.duration")
tool_duration = _metric_points(reader, "gen_ai.tool.duration")
tool_count = _metric_points(reader, "gen_ai.tool.call.count")
assert len(llm_duration) == 1
assert len(llm_count) == 1
assert len(agent_duration) == 1
assert len(tool_duration) == 1
assert len(tool_count) == 1
assert dict(llm_duration[0].attributes) == {"gen_ai.system": "unknown"}
assert dict(llm_count[0].attributes) == {"status": "success"}
assert dict(agent_duration[0].attributes) == {}
assert dict(tool_duration[0].attributes) == {}
assert dict(tool_count[0].attributes) == {}
assert callbacks._metric_state.active_count() == 0
assert callbacks._span_state.active_count() == 0
finally:
await callbacks.unregister(framework)
meter_provider.shutdown()
@pytest.mark.asyncio
async def test_message_policy_keeps_shape_without_content_and_redacts_separately(
telemetry_env: SimpleNamespace,
) -> None:
telemetry_env.callbacks._config = TelemetryConfig(
enabled=True,
log_messages=False,
redact_prompts=True,
redact_completions=False,
)
await telemetry_env.framework.trigger(
LLMCallEvents.LLM_INVOKE_INPUT,
messages=[{"role": "user", "content": "prompt-secret"}],
model="policy-model",
)
result = SimpleNamespace(content="completion-visible", tool_calls=[])
await telemetry_env.framework.trigger(
LLMCallEvents.LLM_INVOKE_OUTPUT,
result=result,
)
telemetry_env.provider.force_flush()
finished = [
item
for item in telemetry_env.exporter.get_finished_spans()
if item.name == "llm.call"
][0]
assert finished.attributes["gen_ai.input.messages.count"] == 1
assert finished.attributes["gen_ai.input.messages.total_length"] == 13
assert "gen_ai.input.messages" not in finished.attributes
assert "gen_ai.output.messages" not in finished.attributes
assert not any(
event.name.startswith("gen_ai.user.message") for event in finished.events
)
assert not any(
event.name == "gen_ai.assistant.message" for event in finished.events
)
@pytest.mark.asyncio
async def test_tool_and_skill_metrics_use_call_identity_and_cleanup(
telemetry_env: SimpleNamespace,
request_metric_channel: None,
) -> None:
del request_metric_channel
inputs = {
"skill_name": "forecast",
"skill_id": "skill-1",
"version": "v2",
"session_id": "session-1",
}
await telemetry_env.framework.trigger(
ToolCallEvents.TOOL_CALL_STARTED,
tool_name="skill_tool",
tool_id="call-skill",
inputs=inputs,
)
assert telemetry_env.callbacks._span_state.active_count() == 1
assert telemetry_env.callbacks._metric_state.active_count() == 1
await telemetry_env.framework.trigger(
ToolCallEvents.TOOL_CALL_FINISHED,
tool_name="skill_tool",
tool_id="call-skill",
inputs=inputs,
result={
"success": True,
"skill": {"name": "forecast", "id": "skill-1", "version": "v2"},
},
)
telemetry_env.provider.force_flush()
assert telemetry_env.callbacks._span_state.active_count() == 0
assert telemetry_env.callbacks._metric_state.active_count() == 0
span = [
item
for item in telemetry_env.exporter.get_finished_spans()
if item.name == "tool.skill_tool"
][0]
assert span.attributes["gen_ai.skill.name"] == "forecast"
assert span.attributes["gen_ai.skill.id"] == "skill-1"
assert span.attributes["gen_ai.skill.version"] == "v2"
assert span.attributes["gen_ai.span.type"] == "tool"
assert span.attributes["gen_ai.operation.name"] == "load_skill"
assert any(event.name == "skill.loaded" for event in span.events)
assert len(_metric_points(telemetry_env.reader, "gen_ai.tool.call.count")) == 1
assert len(_metric_points(telemetry_env.reader, "gen_ai.tool.duration")) == 1
skill_calls = _metric_points(
telemetry_env.reader,
"gen_ai.skill.call.count",
)
assert len(skill_calls) == 1
assert dict(skill_calls[0].attributes) == {
"gen_ai.skill.name": "forecast",
"gen_ai.skill.version": "v2",
"gen_ai.system": "jiuwenclaw",
"jiuwenclaw.channel.id": "web",
}
assert not _metric_points(telemetry_env.reader, "gen_ai.skill.duration")
assert telemetry_env.callbacks._skill_sessions.active_count() == 1
await telemetry_env.framework.trigger(
ToolCallEvents.TOOL_CALL_STARTED,
tool_name="skill_complete",
tool_id="release-skill",
inputs={"skill_name": "forecast", "session_id": "session-1"},
)
await telemetry_env.framework.trigger(
ToolCallEvents.TOOL_CALL_FINISHED,
tool_name="skill_complete",
tool_id="release-skill",
inputs={
"skill_name": "forecast",
"version": "ignored-completion-version",
"session_id": "session-1",
},
result={"success": True},
)
telemetry_env.provider.force_flush()
skill_durations = _metric_points(
telemetry_env.reader,
"gen_ai.skill.duration",
)
assert len(skill_durations) == 1
assert dict(skill_durations[0].attributes) == dict(skill_calls[0].attributes)
assert telemetry_env.callbacks._skill_sessions.active_count() == 0
for tool_id in ("duplicate-start", "duplicate-finish"):
await telemetry_env.framework.trigger(
ToolCallEvents.TOOL_CALL_STARTED
if tool_id.endswith("start")
else ToolCallEvents.TOOL_CALL_FINISHED,
tool_name="skill_complete",
tool_id="duplicate",
inputs={"skill_name": "forecast", "session_id": "session-1"},
**({"result": {"success": True}} if tool_id.endswith("finish") else {}),
)
telemetry_env.provider.force_flush()
assert len(_metric_points(telemetry_env.reader, "gen_ai.skill.duration")) == 1
@pytest.mark.asyncio
@pytest.mark.parametrize(
("tool_name", "operation", "event_name"),
[
("skill_tool", "load_skill", "skill.loaded"),
("skill_complete", "release_skill", "skill.released"),
],
)
async def test_real_tool_wrapper_enriches_skill_output_without_result_identity(
telemetry_env: SimpleNamespace,
tool_name: str,
operation: str,
event_name: str,
) -> None:
tool = _RealSkillLifecycleTool(tool_name)
result = await tool.invoke(
{
"skill_name": "forecast",
"skill_id": "skill-forecast",
"version": "v2",
"relative_file_path": "forecast/SKILL.md",
}
)
telemetry_env.provider.force_flush()
assert isinstance(result, ToolOutput)
assert result.success is True
spans = [
item
for item in telemetry_env.exporter.get_finished_spans()
if item.name == f"tool.{tool_name}"
]
assert len(spans) == 1
span = spans[0]
assert span.attributes["gen_ai.skill.name"] == "forecast"
assert span.attributes["gen_ai.skill.id"] == "skill-forecast"
assert span.attributes["gen_ai.skill.version"] == "v2"
assert span.attributes["gen_ai.operation.name"] == operation
assert [event.name for event in span.events].count(event_name) == 1
assert len(_metric_points(telemetry_env.reader, "gen_ai.skill.call.count")) == (
1 if tool_name == "skill_tool" else 0
)
assert not _metric_points(telemetry_env.reader, "gen_ai.skill.duration")
assert telemetry_env.callbacks._metric_state.active_count() == 0
assert telemetry_env.callbacks._span_state.active_count() == 0
@pytest.mark.asyncio
async def test_real_tool_wrapper_records_skill_error_metrics_from_started_inputs(
telemetry_env: SimpleNamespace,
) -> None:
tool = _RealSkillLifecycleTool(
"skill_tool",
error=RuntimeError("skill load failed"),
)
with pytest.raises(RuntimeError, match="skill load failed"):
await tool.invoke(
{
"skill_name": "forecast",
"skill_id": "skill-forecast",
"version": "v2",
}
)
assert len(_metric_points(telemetry_env.reader, "gen_ai.skill.call.count")) == 1
assert not _metric_points(telemetry_env.reader, "gen_ai.skill.duration")
assert len(_metric_points(telemetry_env.reader, "gen_ai.skill.error.count")) == 1
assert telemetry_env.callbacks._metric_state.active_count() == 0
assert telemetry_env.callbacks._span_state.active_count() == 0
@pytest.mark.asyncio
async def test_unsuccessful_tool_output_records_tool_and_skill_error_metrics(
telemetry_env: SimpleNamespace,
) -> None:
result = await _RealSkillLifecycleTool("skill_tool", success=False).invoke(
{"skill_name": "forecast", "skill_id": "skill-forecast"}
)
assert result.success is False
assert len(_metric_points(telemetry_env.reader, "gen_ai.tool.error.count")) == 1
assert len(_metric_points(telemetry_env.reader, "gen_ai.skill.error.count")) == 1
assert not _metric_points(telemetry_env.reader, "gen_ai.skill.duration")
@pytest.mark.asyncio
async def test_skill_completion_error_reports_error_and_cleans_activation(
telemetry_env: SimpleNamespace,
) -> None:
inputs = {"skill_name": "forecast", "version": "v2", "session_id": "s1"}
await telemetry_env.framework.trigger(
ToolCallEvents.TOOL_CALL_STARTED,
tool_name="skill_tool",
tool_id="load",
inputs=inputs,
)
await telemetry_env.framework.trigger(
ToolCallEvents.TOOL_CALL_FINISHED,
tool_name="skill_tool",
tool_id="load",
inputs=inputs,
result={"success": True},
)
assert telemetry_env.callbacks._skill_sessions.active_count() == 1
await telemetry_env.framework.trigger(
ToolCallEvents.TOOL_CALL_STARTED,
tool_name="skill_complete",
tool_id="release",
inputs=inputs,
)
await telemetry_env.framework.trigger(
ToolCallEvents.TOOL_CALL_ERROR,
tool_name="skill_complete",
tool_id="release",
inputs=inputs,
error=RuntimeError("release failed"),
)
telemetry_env.provider.force_flush()
assert telemetry_env.callbacks._skill_sessions.active_count() == 0
assert len(_metric_points(telemetry_env.reader, "gen_ai.skill.error.count")) == 1
assert not _metric_points(telemetry_env.reader, "gen_ai.skill.duration")
@pytest.mark.asyncio
async def test_unsampled_llm_still_records_metrics_without_a_span() -> None:
reader = InMemoryMetricReader()
meter_provider = MeterProvider(metric_readers=[reader])
callbacks = RichTelemetryCallbacks(
span_registry=SpanRegistryProcessor(max_spans=8, ttl_seconds=60),
metrics=TelemetryMetrics(meter_provider),
config=TelemetryConfig(enabled=True),
)
framework = AsyncCallbackFramework()
await callbacks.register(framework)
try:
await framework.trigger(
LLMCallEvents.LLM_INVOKE_INPUT,
messages=[{"role": "user", "content": "no-span"}],
model="dropped-model",
)
result = SimpleNamespace(
content="answer",
tool_calls=[],
usage_metadata=SimpleNamespace(input_tokens=4, output_tokens=2),
)
returned = await framework.trigger(
LLMCallEvents.LLM_INVOKE_OUTPUT,
result=result,
)
assert returned[0] is result
assert callbacks._span_state.active_count() == 0
assert callbacks._metric_state.active_count() == 0
assert len(_metric_points(reader, "gen_ai.client.operation.count")) == 1
assert len(_metric_points(reader, "gen_ai.client.operation.duration")) == 1
assert len(_metric_points(reader, "gen_ai.client.token.usage")) == 2
assert not _metric_points(reader, "jiuwenclaw.llm.cache_hit_rate")
finally:
await callbacks.unregister(framework)
meter_provider.shutdown()
@pytest.mark.asyncio
async def test_llm_usage_with_cache_information_records_cache_hit_rate() -> None:
reader = InMemoryMetricReader()
meter_provider = MeterProvider(metric_readers=[reader])
callbacks = RichTelemetryCallbacks(
span_registry=SpanRegistryProcessor(max_spans=8, ttl_seconds=60),
metrics=TelemetryMetrics(meter_provider),
config=TelemetryConfig(enabled=True),
)
framework = AsyncCallbackFramework()
await callbacks.register(framework)
try:
await framework.trigger(
LLMCallEvents.LLM_INVOKE_INPUT,
messages=[{"role": "user", "content": "cached"}],
model="cached-model",
)
await framework.trigger(
LLMCallEvents.LLM_INVOKE_OUTPUT,
result=SimpleNamespace(
content="ok",
tool_calls=[],
usage_metadata=SimpleNamespace(
input_tokens=10,
output_tokens=2,
cache_read_tokens=4,
cache_write_tokens=2,
cache_creation_input_tokens=2,
),
),
)
points = _metric_points(reader, "jiuwenclaw.llm.cache_hit_rate")
assert len(points) == 1
assert points[0].sum == pytest.approx(0.4)
finally:
await callbacks.unregister(framework)
meter_provider.shutdown()
@pytest.mark.asyncio
async def test_llm_zero_cache_usage_observed_records_zero_hit_rate() -> None:
reader = InMemoryMetricReader()
meter_provider = MeterProvider(metric_readers=[reader])
callbacks = RichTelemetryCallbacks(
span_registry=SpanRegistryProcessor(max_spans=8, ttl_seconds=60),
metrics=TelemetryMetrics(meter_provider),
config=TelemetryConfig(enabled=True),
)
framework = AsyncCallbackFramework()
await callbacks.register(framework)
try:
await framework.trigger(
LLMCallEvents.LLM_INVOKE_INPUT,
messages=[{"role": "user", "content": "no-cache-hit"}],
model="no-cache-hit-model",
)
await framework.trigger(
LLMCallEvents.LLM_INVOKE_OUTPUT,
result=SimpleNamespace(
content="ok",
tool_calls=[],
usage_metadata=SimpleNamespace(
input_tokens=8,
output_tokens=2,
cache_read_tokens=0,
cache_miss_tokens=8,
cache_write_tokens=0,
),
),
)
points = _metric_points(reader, "jiuwenclaw.llm.cache_hit_rate")
assert len(points) == 1
assert points[0].sum == pytest.approx(0.0)
finally:
await callbacks.unregister(framework)
meter_provider.shutdown()
@pytest.mark.asyncio
async def test_llm_without_cache_usage_skips_hit_rate() -> None:
reader = InMemoryMetricReader()
meter_provider = MeterProvider(metric_readers=[reader])
callbacks = RichTelemetryCallbacks(
span_registry=SpanRegistryProcessor(max_spans=8, ttl_seconds=60),
metrics=TelemetryMetrics(meter_provider),
config=TelemetryConfig(enabled=True),
)
framework = AsyncCallbackFramework()
await callbacks.register(framework)
try:
await framework.trigger(
LLMCallEvents.LLM_INVOKE_INPUT,
messages=[{"role": "user", "content": "cache-unsupported"}],
model="cache-unsupported-model",
)
await framework.trigger(
LLMCallEvents.LLM_INVOKE_OUTPUT,
result=SimpleNamespace(
content="ok",
tool_calls=[],
usage_metadata=SimpleNamespace(input_tokens=8, output_tokens=2),
),
)
assert not _metric_points(reader, "jiuwenclaw.llm.cache_hit_rate")
finally:
await callbacks.unregister(framework)
meter_provider.shutdown()
@pytest.mark.asyncio
async def test_unsampled_enterprise_metric_labels_match_sampled_contract(
unsampled_telemetry_env: SimpleNamespace,
request_metric_channel: None,
request_metric_modes: None,
) -> None:
del request_metric_channel, request_metric_modes
await unsampled_telemetry_env.framework.trigger(
AgentEvents.AGENT_INVOKE_INPUT,
{
"query": "unsampled",
"agent_name": "unsampled-agent",
},
)
await unsampled_telemetry_env.framework.trigger(
LLMCallEvents.LLM_INVOKE_INPUT,
messages=[{"role": "user", "content": "hello"}],
model="unsampled-model",
model_client_config={"client_provider": "OpenAI"},
)
await unsampled_telemetry_env.framework.trigger(
LLMCallEvents.LLM_INVOKE_OUTPUT,
result=SimpleNamespace(
content="ok",
tool_calls=[],
usage_metadata=SimpleNamespace(input_tokens=4, output_tokens=2),
),
)
await unsampled_telemetry_env.framework.trigger(
ToolCallEvents.TOOL_CALL_STARTED,
tool_name="weather",
tool_id="unsampled-call",
inputs={},
)
await unsampled_telemetry_env.framework.trigger(
ToolCallEvents.TOOL_CALL_ERROR,
tool_name="weather",
tool_id="unsampled-call",
error=RuntimeError("tool failed"),
)
await unsampled_telemetry_env.framework.trigger(
AgentEvents.AGENT_INVOKE_OUTPUT,
result="done",
)
agent_points = _metric_points(
unsampled_telemetry_env.reader,
"jiuwenclaw.agent.duration",
)
llm_duration_points = _metric_points(
unsampled_telemetry_env.reader,
"gen_ai.client.operation.duration",
)
llm_count_points = _metric_points(
unsampled_telemetry_env.reader,
"gen_ai.client.operation.count",
)
token_points = _metric_points(
unsampled_telemetry_env.reader,
"gen_ai.client.token.usage",
)
assert len(agent_points) == 1
assert len(llm_duration_points) == 1
assert len(llm_count_points) == 1
assert len(token_points) == 2
assert dict(agent_points[0].attributes) == {
"jiuwenclaw.agent.name": "unsampled-agent",
"jiuwenclaw.channel.id": "web",
"jiuwenclaw.mode": "agent",
"jiuwenclaw.adapter.mode": "flash",
}
assert dict(llm_duration_points[0].attributes) == {
"gen_ai.request.model": "unsampled-model",
"gen_ai.system": "openai",
"jiuwenclaw.channel.id": "web",
"jiuwenclaw.mode": "agent",
"jiuwenclaw.adapter.mode": "flash",
}
assert dict(llm_count_points[0].attributes) == {
"gen_ai.request.model": "unsampled-model",
"status": "success",
"jiuwenclaw.channel.id": "web",
"jiuwenclaw.mode": "agent",
"jiuwenclaw.adapter.mode": "flash",
}
assert {
point.attributes["gen_ai.token.type"]: dict(point.attributes)
for point in token_points
} == {
"input": {
"gen_ai.request.model": "unsampled-model",
"gen_ai.system": "openai",
"jiuwenclaw.channel.id": "web",
"jiuwenclaw.mode": "agent",
"jiuwenclaw.adapter.mode": "flash",
"gen_ai.token.type": "input",
},
"output": {
"gen_ai.request.model": "unsampled-model",
"gen_ai.system": "openai",
"jiuwenclaw.channel.id": "web",
"jiuwenclaw.mode": "agent",
"jiuwenclaw.adapter.mode": "flash",
"gen_ai.token.type": "output",
},
}
expected_tool_attributes = {
"gen_ai.tool.name": "weather",
"jiuwenclaw.channel.id": "web",
"jiuwenclaw.mode": "agent",
"jiuwenclaw.adapter.mode": "flash",
}
for metric_name in (
"gen_ai.tool.duration",
"gen_ai.tool.call.count",
"gen_ai.tool.error.count",
):
points = _metric_points(unsampled_telemetry_env.reader, metric_name)
assert len(points) == 1
assert dict(points[0].attributes) == expected_tool_attributes
assert "gen_ai.tool.call.id" not in points[0].attributes
assert unsampled_telemetry_env.exporter.get_finished_spans() == ()
assert unsampled_telemetry_env.callbacks._metric_state.active_count() == 0
assert unsampled_telemetry_env.callbacks._span_state.active_count() == 0
await unsampled_telemetry_env.framework.trigger(
LLMCallEvents.LLM_INVOKE_INPUT,
messages=[],
model="after-agent",
model_client_config={"client_provider": "OpenAI"},
)
await unsampled_telemetry_env.framework.trigger(
LLMCallEvents.LLM_INVOKE_OUTPUT,
result=SimpleNamespace(content="ok", tool_calls=[]),
)
after_agent = [
point
for point in _metric_points(
unsampled_telemetry_env.reader,
"gen_ai.client.operation.duration",
)
if point.attributes.get("gen_ai.request.model") == "after-agent"
]
assert len(after_agent) == 1
assert dict(after_agent[0].attributes) == {
"gen_ai.request.model": "after-agent",
"gen_ai.system": "openai",
"jiuwenclaw.channel.id": "web",
"jiuwenclaw.mode": "agent",
"jiuwenclaw.adapter.mode": "flash",
}
@pytest.mark.asyncio
@pytest.mark.parametrize("error_type", [RuntimeError, asyncio.CancelledError])
async def test_request_channel_survives_unpaired_agent_failure_for_later_metrics(
error_type: type[BaseException],
) -> None:
metrics = _RecordingMetrics()
callbacks = RichTelemetryCallbacks(
span_registry=SpanRegistryProcessor(max_spans=8, ttl_seconds=60),
metrics=metrics,
config=TelemetryConfig(enabled=True),
)
channel_token = metrics_channel_id.set("request-channel")
async def failing_agent() -> None:
await callbacks._on_agent_input(
{"agent_name": "inner-agent"},
)
raise error_type("business failure")
try:
with pytest.raises(error_type):
await failing_agent()
await callbacks._on_llm_invoke_input(
messages=[],
model="outer-model",
model_client_config={"provider": "OpenAI"},
)
await callbacks._on_llm_invoke_output(
result=SimpleNamespace(content="ok", tool_calls=[]),
)
await callbacks._on_tool_input(
tool_name="outer-tool",
tool_id="outer-call",
inputs={},
)
await callbacks._on_tool_error(
tool_name="outer-tool",
tool_id="outer-call",
error=RuntimeError("tool failed"),
)
finally:
metrics_channel_id.reset(channel_token)
assert [
call[3]
for call in metrics.calls
if call[1] == "gen_ai.client.operation.duration"
] == [
{
"gen_ai.request.model": "outer-model",
"gen_ai.system": "openai",
"jiuwenclaw.channel.id": "request-channel",
}
]
assert [
call[3]
for call in metrics.calls
if call[1] == "gen_ai.tool.error.count"
] == [
{
"gen_ai.tool.name": "outer-tool",
"jiuwenclaw.channel.id": "request-channel",
}
]
assert not hasattr(callbacks, "_agent_label_states")
@pytest.mark.asyncio
async def test_agent_and_common_attributes_and_parent_token_totals(
telemetry_env: SimpleNamespace,
) -> None:
identity_token = IdentityStore.set_identity(
IdentityInfo(user_id="user-1", domain_id="domain-1", app_id="app-1")
)
session = SimpleNamespace(get_session_id=lambda: "session-1")
telemetry_env.parent.set_attribute("agentteam.member.name", "main-agent")
try:
await telemetry_env.framework.trigger(
AgentEvents.AGENT_INVOKE_INPUT,
{
"query": "what is the weather",
"conversation_id": "conversation-1",
"mode": "react",
"request_id": "request-1",
"channel_id": "web",
"parent_session_id": "parent-1",
"iteration": 3,
},
session=session,
)
await telemetry_env.framework.trigger(
LLMCallEvents.LLM_INVOKE_INPUT,
messages=[{"role": "user", "content": "weather"}],
model="model",
)
await telemetry_env.framework.trigger(
LLMCallEvents.LLM_INVOKE_OUTPUT,
result=SimpleNamespace(
content="sunny",
tool_calls=[],
usage_metadata=SimpleNamespace(input_tokens=9, output_tokens=4),
),
)
agent_result = object()
returned = await telemetry_env.framework.trigger(
AgentEvents.AGENT_INVOKE_OUTPUT,
result=agent_result,
)
finally:
IdentityStore.clear(identity_token)
assert returned[0] is agent_result
attrs = telemetry_env.parent.attributes
assert attrs["gen_ai.conversation.id"] == "conversation-1"
assert attrs["jiuwenclaw.session.id"] == "session-1"
assert attrs["jiuwenclaw.request.id"] == "request-1"
assert attrs["jiuwenclaw.channel.id"] == "web"
assert attrs["jiuwenclaw.agent.parent"] == "parent-1"
assert attrs["jiuwenclaw.iteration"] == 3
assert attrs["jiuwenclaw.agent.mode"] == "react"
assert attrs["gen_ai.agent.name"] == "main-agent"
assert attrs["jiuwenclaw.agent.name"] == "main-agent"
assert attrs["gen_ai.span.type"] == "agent"
assert attrs["user.id"] == "user-1"
assert attrs["domain.id"] == "domain-1"
assert attrs["app.id"] == "app-1"
assert attrs["jiuwenclaw.user.id"] == "user-1"
assert attrs["jiuwenclaw.claw.id"] == "claw-config"
assert attrs["service.version"] == "1.2.3"
assert attrs["gen_ai.usage.input_tokens"] == 9
assert attrs["gen_ai.usage.output_tokens"] == 4
duration_points = _metric_points(
telemetry_env.reader,
"jiuwenclaw.agent.duration",
)
assert len(duration_points) == 1
assert dict(duration_points[0].attributes) == {
"jiuwenclaw.agent.name": "main-agent",
"jiuwenclaw.channel.id": "web",
"user_id": "user-1",
"domain_id": "domain-1",
"app_id": "app-1",
}
@pytest.mark.asyncio
@pytest.mark.parametrize(
("redact_prompts", "redact_completions", "input_expected", "output_expected"),
[
(True, False, "[REDACTED]", "completion-secret"),
(False, True, "prompt-secret", "[REDACTED]"),
],
)
async def test_prompt_and_completion_redaction_are_independent(
telemetry_env: SimpleNamespace,
redact_prompts: bool,
redact_completions: bool,
input_expected: str,
output_expected: str,
) -> None:
telemetry_env.callbacks._config = TelemetryConfig(
enabled=True,
log_messages=True,
redact_prompts=redact_prompts,
redact_completions=redact_completions,
)
await telemetry_env.framework.trigger(
LLMCallEvents.LLM_INVOKE_INPUT,
messages=[{"role": "user", "content": "prompt-secret"}],
model="redaction-model",
tools=[
{
"function": {
"name": "weather",
"description": "tool-secret",
}
}
],
)
await telemetry_env.framework.trigger(
LLMCallEvents.LLM_INVOKE_OUTPUT,
result=SimpleNamespace(content="completion-secret", tool_calls=[]),
)
telemetry_env.provider.force_flush()
span = [
item
for item in telemetry_env.exporter.get_finished_spans()
if item.name == "llm.call"
][0]
assert input_expected in span.attributes["gen_ai.input.messages"]
assert output_expected in span.attributes["gen_ai.output.messages"]
if redact_prompts:
assert "prompt-secret" not in span.attributes["gen_ai.input.messages"]
if redact_completions:
assert "completion-secret" not in span.attributes["gen_ai.output.messages"]
assert "gen_ai.tool.definitions" in span.attributes
assistant_events = [
event for event in span.events if event.name == "gen_ai.assistant.message"
]
assert len(assistant_events) == 1
assert output_expected in assistant_events[0].attributes["content"]
@pytest.mark.asyncio
async def test_completion_redaction_hides_tool_call_arguments(
telemetry_env: SimpleNamespace,
) -> None:
telemetry_env.callbacks._config = TelemetryConfig(
enabled=True,
log_messages=True,
redact_completions=True,
attribute_value_max_length=256,
)
await telemetry_env.framework.trigger(
LLMCallEvents.LLM_STREAM_INPUT,
[{"role": "user", "content": "call a tool"}],
model="redaction-model",
)
await telemetry_env.framework.trigger(
LLMCallEvents.LLM_OUTPUT,
is_stream=True,
response="completion-secret",
reasoning_content="reasoning-secret",
tool_calls=[
{
"id": "call-secret",
"function": {
"name": "private_tool",
"arguments": '{"password":"super-secret"}',
},
}
],
)
telemetry_env.provider.force_flush()
span = [
item
for item in telemetry_env.exporter.get_finished_spans()
if item.name == "llm.call"
][0]
output = span.attributes["gen_ai.output.messages"]
assert "completion-secret" not in output
assert "reasoning-secret" not in output
assert "super-secret" not in output
assert "private_tool" in output
assert "[REDACTED]" in output
assistant_events = [
event for event in span.events if event.name == "gen_ai.assistant.message"
]
assert len(assistant_events) == 1
event_content = assistant_events[0].attributes["content"]
assert event_content == output
assert len(event_content) <= 256
assert "completion-secret" not in event_content
assert "reasoning-secret" not in event_content
assert "super-secret" not in event_content
@pytest.mark.asyncio
async def test_log_messages_false_suppresses_tool_payload_attributes_and_events(
telemetry_env: SimpleNamespace,
) -> None:
telemetry_env.callbacks._config = TelemetryConfig(
enabled=True,
log_messages=False,
)
await telemetry_env.framework.trigger(
ToolCallEvents.TOOL_CALL_STARTED,
tool_name="private_tool",
tool_id="private-call",
inputs={"password": "prompt-secret"},
)
await telemetry_env.framework.trigger(
ToolCallEvents.TOOL_CALL_FINISHED,
tool_name="private_tool",
tool_id="private-call",
result={"token": "completion-secret"},
)
telemetry_env.provider.force_flush()
span = [
item
for item in telemetry_env.exporter.get_finished_spans()
if item.name == "tool.private_tool"
][0]
assert span.attributes["gen_ai.tool.name"] == "private_tool"
assert span.attributes["gen_ai.span.type"] == "tool"
assert "gen_ai.tool.arguments" not in span.attributes
assert "gen_ai.tool.result" not in span.attributes
assert not any(
event.name in {"tool.arguments", "tool.result"} for event in span.events
)
assert len(_metric_points(telemetry_env.reader, "gen_ai.tool.call.count")) == 1
@pytest.mark.asyncio
async def test_llm_error_records_metrics_and_cleans_both_state_registries(
telemetry_env: SimpleNamespace,
) -> None:
await telemetry_env.framework.trigger(
LLMCallEvents.LLM_INVOKE_INPUT,
messages=[{"role": "user", "content": "fail"}],
model="error-model",
)
error = RuntimeError("model failed")
await telemetry_env.framework.trigger(LLMCallEvents.LLM_CALL_ERROR, error=error)
telemetry_env.provider.force_flush()
assert telemetry_env.callbacks._span_state.active_count() == 0
assert telemetry_env.callbacks._metric_state.active_count() == 0
span = [
item
for item in telemetry_env.exporter.get_finished_spans()
if item.name == "llm.call"
][0]
assert span.attributes["error.type"] == "RuntimeError"
count_points = _metric_points(telemetry_env.reader, "gen_ai.client.operation.count")
assert len(count_points) == 1
assert count_points[0].value == 1
assert (
len(_metric_points(telemetry_env.reader, "gen_ai.client.operation.duration"))
== 1
)
@pytest.mark.asyncio
async def test_llm_cancellation_marks_the_core_span_and_cleans_state(
telemetry_env: SimpleNamespace,
) -> None:
await telemetry_env.framework.trigger(
LLMCallEvents.LLM_INVOKE_INPUT,
messages=[],
model="cancel-model",
)
await telemetry_env.framework.trigger(
LLMCallEvents.LLM_CALL_ERROR,
error=asyncio.CancelledError(),
)
telemetry_env.provider.force_flush()
span = [
item
for item in telemetry_env.exporter.get_finished_spans()
if item.name == "llm.call"
][0]
assert span.attributes["error.type"] == "CancelledError"
assert span.attributes["jiuwenclaw.canceled"] is True
assert telemetry_env.callbacks._metric_state.active_count() == 0
assert telemetry_env.callbacks._span_state.active_count() == 0
@pytest.mark.asyncio
async def test_common_enrichment_does_not_overwrite_existing_core_attributes(
telemetry_env: SimpleNamespace,
) -> None:
telemetry_env.parent.set_attribute("user.id", "core-user")
telemetry_env.parent.set_attribute("jiuwenclaw.user.id", "stale-alias")
token = IdentityStore.set_identity(
IdentityInfo(user_id="enriched-user", domain_id="domain", app_id="app")
)
try:
await telemetry_env.framework.trigger(
AgentEvents.AGENT_INVOKE_INPUT,
{"query": "hello"},
)
await telemetry_env.framework.trigger(
AgentEvents.AGENT_INVOKE_OUTPUT,
result="done",
)
finally:
IdentityStore.clear(token)
assert telemetry_env.parent.attributes["user.id"] == "core-user"
assert telemetry_env.parent.attributes["jiuwenclaw.user.id"] == "core-user"
@pytest.mark.asyncio
async def test_skill_release_sets_enterprise_operation_aliases(
telemetry_env: SimpleNamespace,
) -> None:
inputs = {"skill_name": "forecast", "skill_id": "skill-1"}
await telemetry_env.framework.trigger(
ToolCallEvents.TOOL_CALL_STARTED,
tool_name="skill_complete",
tool_id="release-call",
inputs=inputs,
)
await telemetry_env.framework.trigger(
ToolCallEvents.TOOL_CALL_FINISHED,
tool_name="skill_complete",
tool_id="release-call",
inputs=inputs,
result={"success": True},
)
telemetry_env.provider.force_flush()
span = [
item
for item in telemetry_env.exporter.get_finished_spans()
if item.name == "tool.skill_complete"
][0]
assert span.attributes["gen_ai.span.type"] == "tool"
assert span.attributes["gen_ai.operation.name"] == "release_skill"
assert span.attributes["gen_ai.skill.name"] == "forecast"
assert any(event.name == "skill.released" for event in span.events)
assert not _metric_points(telemetry_env.reader, "gen_ai.skill.duration")
@pytest.mark.asyncio
async def test_real_agent_stream_only_finalizes_on_last_chunk(
telemetry_env: SimpleNamespace,
) -> None:
first = SimpleNamespace(index=0, last_chunk=False)
final = SimpleNamespace(index=1, last_chunk=True)
async def stream(inputs):
assert inputs["query"] == "stream query"
yield first
yield final
wrapped = _decorate_like_real_agent_stream(
telemetry_env.framework,
stream,
)
iterator = wrapped({"query": "stream query"})
assert await anext(iterator) is first
assert telemetry_env.callbacks._metric_state.active_count() == 1
assert telemetry_env.callbacks._span_state.active_count() == 1
assert not _metric_points(telemetry_env.reader, "jiuwenclaw.agent.duration")
assert await anext(iterator) is final
with pytest.raises(StopAsyncIteration):
await anext(iterator)
assert telemetry_env.callbacks._metric_state.active_count() == 0
assert telemetry_env.callbacks._span_state.active_count() == 0
assert len(_metric_points(telemetry_env.reader, "jiuwenclaw.agent.duration")) == 1
@pytest.mark.asyncio
@pytest.mark.parametrize("failure_point", ["serializer", "event"])
async def test_llm_input_field_failure_keeps_state_until_normal_output(
telemetry_env: SimpleNamespace,
monkeypatch: pytest.MonkeyPatch,
failure_point: str,
) -> None:
warnings: list[str] = []
monkeypatch.setattr(
callbacks_module._LOGGER,
"warning",
lambda message, **kwargs: warnings.append(message),
)
if failure_point == "serializer":
def fail_serializer(*args, **kwargs):
del args, kwargs
raise RuntimeError("input serialization failed")
monkeypatch.setattr(
callbacks_module,
"serialize_input_messages",
fail_serializer,
)
else:
span_type = type(telemetry_env.parent)
original_add_event = span_type.add_event
def fail_input_event(self, name, *args, **kwargs):
if name == "gen_ai.user.message":
raise RuntimeError("input event failed")
return original_add_event(self, name, *args, **kwargs)
monkeypatch.setattr(span_type, "add_event", fail_input_event)
await telemetry_env.framework.trigger(
LLMCallEvents.LLM_INVOKE_INPUT,
messages=[{"role": "user", "content": "hello"}],
model="resilient-model",
)
assert warnings == ["LLM telemetry input enrichment failed"]
assert telemetry_env.callbacks._metric_state.active_count() == 1
assert telemetry_env.callbacks._span_state.active_count() == 1
result = SimpleNamespace(
content="world",
finish_reason="stop",
tool_calls=[],
usage_metadata=SimpleNamespace(
input_tokens=7,
output_tokens=3,
cache_read_input_tokens=2,
cache_creation_input_tokens=4,
reasoning_tokens=1,
),
)
returned = await telemetry_env.framework.trigger(
LLMCallEvents.LLM_INVOKE_OUTPUT,
result=result,
)
telemetry_env.provider.force_flush()
assert returned[0] is result
assert telemetry_env.callbacks._metric_state.active_count() == 0
assert telemetry_env.callbacks._span_state.active_count() == 0
span = [
item
for item in telemetry_env.exporter.get_finished_spans()
if item.name == "llm.call"
][0]
assert span.attributes["gen_ai.response.finish_reason"] == "stop"
assert span.attributes["gen_ai.usage.input_tokens"] == 7
assert span.attributes["gen_ai.usage.output_tokens"] == 3
assert span.attributes["gen_ai.usage.cache_read.input_tokens"] == 2
assert span.attributes["gen_ai.usage.cache_creation.input_tokens"] == 4
assert span.attributes["gen_ai.usage.reasoning.output_tokens"] == 1
assert telemetry_env.parent.attributes["gen_ai.usage.input_tokens"] == 7
assert telemetry_env.parent.attributes["gen_ai.usage.output_tokens"] == 3
assert (
len(_metric_points(telemetry_env.reader, "gen_ai.client.operation.count")) == 1
)
assert (
len(_metric_points(telemetry_env.reader, "gen_ai.client.operation.duration"))
== 1
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"failure_point",
["arguments", "skill", "attribute"],
)
async def test_tool_input_field_failure_keeps_state_until_normal_output(
telemetry_env: SimpleNamespace,
monkeypatch: pytest.MonkeyPatch,
failure_point: str,
) -> None:
warnings: list[str] = []
monkeypatch.setattr(
callbacks_module._LOGGER,
"warning",
lambda message, **kwargs: warnings.append(message),
)
restore = None
if failure_point == "arguments":
original = telemetry_env.callbacks._bounded_json
def fail_arguments(*args, **kwargs):
del args, kwargs
raise RuntimeError("tool argument serialization failed")
monkeypatch.setattr(telemetry_env.callbacks, "_bounded_json", fail_arguments)
def restore():
monkeypatch.setattr(
telemetry_env.callbacks,
"_bounded_json",
original,
)
elif failure_point == "skill":
original = callbacks_module.extract_skill
def fail_skill(*args, **kwargs):
del args, kwargs
raise RuntimeError("skill extraction failed")
monkeypatch.setattr(callbacks_module, "extract_skill", fail_skill)
def restore():
monkeypatch.setattr(
callbacks_module,
"extract_skill",
original,
)
else:
original = telemetry_env.callbacks._set_if_empty
def fail_attribute(span, key, value):
if key == "gen_ai.tool.call.id":
raise RuntimeError("tool attribute failed")
return original(span, key, value)
monkeypatch.setattr(
telemetry_env.callbacks,
"_set_if_empty",
fail_attribute,
)
def restore():
monkeypatch.setattr(
telemetry_env.callbacks,
"_set_if_empty",
original,
)
inputs = {"skill_name": "forecast", "skill_id": "skill-1"}
await telemetry_env.framework.trigger(
ToolCallEvents.TOOL_CALL_STARTED,
tool_name="skill_tool",
tool_id="resilient-tool-call",
inputs=inputs,
)
assert warnings == ["Tool telemetry input enrichment failed"]
assert telemetry_env.callbacks._metric_state.active_count() == 1
assert telemetry_env.callbacks._span_state.active_count() == 1
assert restore is not None
restore()
result = {"success": True}
returned = await telemetry_env.framework.trigger(
ToolCallEvents.TOOL_CALL_FINISHED,
tool_name="skill_tool",
tool_id="resilient-tool-call",
inputs=inputs,
result=result,
)
assert returned[0] is result
assert telemetry_env.callbacks._metric_state.active_count() == 0
assert telemetry_env.callbacks._span_state.active_count() == 0
assert len(_metric_points(telemetry_env.reader, "gen_ai.tool.call.count")) == 1
assert len(_metric_points(telemetry_env.reader, "gen_ai.tool.duration")) == 1
@pytest.mark.asyncio
async def test_agent_input_field_failure_keeps_state_until_normal_output(
telemetry_env: SimpleNamespace,
monkeypatch: pytest.MonkeyPatch,
) -> None:
warnings: list[str] = []
def fail_agent_name(*args, **kwargs):
del args, kwargs
raise RuntimeError("agent name enrichment failed")
monkeypatch.setattr(
telemetry_env.callbacks,
"_write_agent_name",
fail_agent_name,
)
monkeypatch.setattr(
callbacks_module._LOGGER,
"warning",
lambda message, **kwargs: warnings.append(message),
)
await telemetry_env.framework.trigger(
AgentEvents.AGENT_INVOKE_INPUT,
{"query": "hello", "conversation_id": "conversation-1"},
)
assert warnings == ["Agent telemetry input enrichment failed"]
assert telemetry_env.callbacks._metric_state.active_count() == 1
assert telemetry_env.callbacks._span_state.active_count() == 1
result = object()
returned = await telemetry_env.framework.trigger(
AgentEvents.AGENT_INVOKE_OUTPUT,
result=result,
)
assert returned[0] is result
assert telemetry_env.callbacks._metric_state.active_count() == 0
assert telemetry_env.callbacks._span_state.active_count() == 0
assert len(_metric_points(telemetry_env.reader, "jiuwenclaw.agent.duration")) == 1
@pytest.mark.asyncio
async def test_nested_agent_callbacks_finish_each_call_in_lifo_order(
telemetry_env: SimpleNamespace,
) -> None:
await telemetry_env.callbacks._on_agent_input(
{"query": "outer", "conversation_id": "outer-conversation"}
)
await telemetry_env.callbacks._on_agent_input(
{"query": "inner", "conversation_id": "inner-conversation"}
)
assert telemetry_env.callbacks._metric_state.active_count() == 2
inner_result = object()
assert (
await telemetry_env.callbacks._on_agent_invoke_output(result=inner_result)
is inner_result
)
assert telemetry_env.callbacks._metric_state.active_count() == 1
outer_result = object()
assert (
await telemetry_env.callbacks._on_agent_invoke_output(result=outer_result)
is outer_result
)
assert telemetry_env.callbacks._metric_state.active_count() == 0
assert telemetry_env.callbacks._span_state.active_count() == 0
duration_points = _metric_points(
telemetry_env.reader,
"jiuwenclaw.agent.duration",
)
assert sum(point.count for point in duration_points) == 2
@pytest.mark.asyncio
@pytest.mark.parametrize("failure_point", ["serializer", "assistant_event"])
async def test_llm_output_field_failure_preserves_remaining_enrichment_and_metrics(
telemetry_env: SimpleNamespace,
monkeypatch: pytest.MonkeyPatch,
failure_point: str,
) -> None:
await telemetry_env.framework.trigger(
LLMCallEvents.LLM_INVOKE_INPUT,
messages=[{"role": "user", "content": "hello"}],
model="resilient-output-model",
)
warnings: list[str] = []
monkeypatch.setattr(
callbacks_module._LOGGER,
"warning",
lambda message, **kwargs: warnings.append(message),
)
if failure_point == "serializer":
def fail_serializer(*args, **kwargs):
del args, kwargs
raise RuntimeError("output serialization failed")
monkeypatch.setattr(
callbacks_module,
"serialize_output_message",
fail_serializer,
)
else:
span_type = type(telemetry_env.parent)
original_add_event = span_type.add_event
def fail_assistant_event(self, name, *args, **kwargs):
if name == "gen_ai.assistant.message":
raise RuntimeError("assistant event failed")
return original_add_event(self, name, *args, **kwargs)
monkeypatch.setattr(span_type, "add_event", fail_assistant_event)
result = SimpleNamespace(
content="world",
finish_reason="stop",
tool_calls=[],
usage_metadata=SimpleNamespace(
input_tokens=11,
output_tokens=5,
cache_read_input_tokens=3,
cache_creation_input_tokens=2,
reasoning_tokens=1,
),
)
returned = await telemetry_env.framework.trigger(
LLMCallEvents.LLM_INVOKE_OUTPUT,
result=result,
)
telemetry_env.provider.force_flush()
assert returned[0] is result
assert warnings == ["LLM telemetry output enrichment failed"]
assert telemetry_env.callbacks._metric_state.active_count() == 0
assert telemetry_env.callbacks._span_state.active_count() == 0
span = [
item
for item in telemetry_env.exporter.get_finished_spans()
if item.name == "llm.call"
][0]
assert span.attributes["gen_ai.response.finish_reason"] == "stop"
assert span.attributes["gen_ai.usage.input_tokens"] == 11
assert span.attributes["gen_ai.usage.output_tokens"] == 5
assert span.attributes["gen_ai.usage.cache_read.input_tokens"] == 3
assert span.attributes["gen_ai.usage.cache_creation.input_tokens"] == 2
assert span.attributes["gen_ai.usage.reasoning.output_tokens"] == 1
assert telemetry_env.parent.attributes["gen_ai.usage.input_tokens"] == 11
assert telemetry_env.parent.attributes["gen_ai.usage.output_tokens"] == 5
output_messages = span.attributes.get("gen_ai.output.messages")
assistant_events = [
event for event in span.events if event.name == "gen_ai.assistant.message"
]
if failure_point == "serializer":
assert output_messages is None
else:
assert "world" in output_messages
assert assistant_events == []
assert (
len(_metric_points(telemetry_env.reader, "gen_ai.client.operation.count")) == 1
)
assert (
len(_metric_points(telemetry_env.reader, "gen_ai.client.operation.duration"))
== 1
)
token_points = {
point.attributes["gen_ai.token.type"]: point.value
for point in _metric_points(
telemetry_env.reader,
"gen_ai.client.token.usage",
)
}
assert token_points == {
"input": 11,
"output": 5,
"cache_read": 3,
"cache_creation": 2,
"reasoning": 1,
}
@pytest.mark.asyncio
async def test_sequential_llm_usage_overrides_inherited_parent_totals(
telemetry_env: SimpleNamespace,
) -> None:
for index, (input_tokens, output_tokens) in enumerate(((7, 2), (11, 4))):
call_id = f"sequential-{index}"
model = f"sequential-model-{index}"
await telemetry_env.framework.trigger(
LLMCallEvents.LLM_INVOKE_INPUT,
messages=[{"role": "user", "content": f"prompt-{index}"}],
model=model,
call_id=call_id,
)
await telemetry_env.framework.trigger(
LLMCallEvents.LLM_INVOKE_OUTPUT,
result=SimpleNamespace(
content=f"result-{index}",
finish_reason="stop",
tool_calls=[],
usage_metadata=SimpleNamespace(
input_tokens=input_tokens,
output_tokens=output_tokens,
),
),
call_id=call_id,
)
telemetry_env.provider.force_flush()
spans_by_model = {
span.attributes["gen_ai.request.model"]: span
for span in telemetry_env.exporter.get_finished_spans()
if span.name == "llm.call"
}
second_span = spans_by_model["sequential-model-1"]
assert second_span.attributes["gen_ai.usage.input_tokens"] == 11
assert second_span.attributes["gen_ai.usage.output_tokens"] == 4
assert second_span.attributes["gen_ai.usage.total_tokens"] == 15
@pytest.mark.asyncio
@pytest.mark.parametrize("callback_kind", ["tool", "agent"])
@pytest.mark.parametrize(
"error_type",
[RuntimeError, asyncio.CancelledError, KeyboardInterrupt, SystemExit],
)
async def test_tool_and_agent_input_failures_preserve_or_cleanup_state(
telemetry_env: SimpleNamespace,
monkeypatch: pytest.MonkeyPatch,
callback_kind: str,
error_type: type[BaseException],
) -> None:
error = error_type("enrichment failed")
warnings: list[str] = []
def fail_common(span):
del span
raise error
monkeypatch.setattr(
telemetry_env.callbacks,
"_write_common_attributes",
fail_common,
)
monkeypatch.setattr(
callbacks_module._LOGGER,
"warning",
lambda message, **kwargs: warnings.append(message),
)
if callback_kind == "tool":
monkeypatch.setattr(
telemetry_env.callbacks,
"_find_tool_span",
lambda kwargs: telemetry_env.parent,
)
call = telemetry_env.callbacks._on_tool_input(
tool_name="failing_tool",
tool_id="failing-call",
inputs={"value": 1},
)
else:
call = telemetry_env.callbacks._on_agent_input({"query": "hello"})
if issubclass(error_type, Exception):
await call
assert warnings == [
f"{callback_kind.title()} telemetry input enrichment failed"
]
assert telemetry_env.callbacks._metric_state.active_count() == 1
assert telemetry_env.callbacks._span_state.active_count() == 1
if callback_kind == "tool":
result = object()
returned = await telemetry_env.callbacks._on_tool_output(
tool_name="failing_tool",
tool_id="failing-call",
inputs={"value": 1},
result=result,
)
assert returned is result
assert (
len(_metric_points(telemetry_env.reader, "gen_ai.tool.call.count")) == 1
)
assert (
len(_metric_points(telemetry_env.reader, "gen_ai.tool.duration")) == 1
)
else:
result = object()
returned = await telemetry_env.callbacks._on_agent_invoke_output(
result=result,
)
assert returned is result
assert (
len(_metric_points(telemetry_env.reader, "jiuwenclaw.agent.duration"))
== 1
)
else:
with pytest.raises(error_type):
await call
assert warnings == []
assert telemetry_env.callbacks._metric_state.active_count() == 0
assert telemetry_env.callbacks._span_state.active_count() == 0
@pytest.mark.asyncio
async def test_tool_input_failure_before_state_creation_degrades_cleanly(
telemetry_env: SimpleNamespace,
monkeypatch: pytest.MonkeyPatch,
) -> None:
warnings: list[str] = []
class BrokenToolName:
def __str__(self) -> str:
raise RuntimeError("broken tool name")
monkeypatch.setattr(
callbacks_module._LOGGER,
"warning",
lambda message, **kwargs: warnings.append(message),
)
await telemetry_env.callbacks._on_tool_input(
tool_name=BrokenToolName(),
tool_id="broken-call",
inputs={},
)
assert warnings == ["Tool telemetry input enrichment failed"]
assert telemetry_env.callbacks._metric_state.active_count() == 0
assert telemetry_env.callbacks._span_state.active_count() == 0
@pytest.mark.asyncio
async def test_agent_stream_control_failure_cleans_state_before_propagating(
telemetry_env: SimpleNamespace,
) -> None:
class BrokenFinalChunk:
@property
def last_chunk(self) -> bool:
raise KeyboardInterrupt("control")
await telemetry_env.callbacks._on_agent_input({"query": "stream"})
with pytest.raises(KeyboardInterrupt):
await telemetry_env.callbacks._on_agent_stream_output(
result=BrokenFinalChunk(),
)
assert telemetry_env.callbacks._metric_state.active_count() == 0
assert telemetry_env.callbacks._span_state.active_count() == 0
@pytest.mark.asyncio
async def test_llm_first_chunk_ordinary_failure_degrades_and_preserves_chunk(
telemetry_env: SimpleNamespace,
monkeypatch: pytest.MonkeyPatch,
) -> None:
await telemetry_env.framework.trigger(
LLMCallEvents.LLM_STREAM_INPUT,
[{"role": "user", "content": "stream"}],
model="fragile-stream-model",
)
original_span_key = telemetry_env.callbacks._span_key
def fail_span_key(span):
del span
raise RuntimeError("first chunk enrichment failed")
warnings: list[str] = []
monkeypatch.setattr(telemetry_env.callbacks, "_span_key", fail_span_key)
monkeypatch.setattr(
callbacks_module._LOGGER,
"warning",
lambda message, **kwargs: warnings.append(message),
)
chunk = SimpleNamespace(content="first")
returned = await telemetry_env.callbacks._on_llm_stream_output(result=chunk)
assert returned is chunk
assert warnings == ["LLM telemetry stream output enrichment failed"]
assert telemetry_env.callbacks._metric_state.active_count() == 1
assert telemetry_env.callbacks._span_state.active_count() == 1
monkeypatch.setattr(telemetry_env.callbacks, "_span_key", original_span_key)
await telemetry_env.framework.trigger(
LLMCallEvents.LLM_OUTPUT,
is_stream=True,
response="done",
usage=SimpleNamespace(input_tokens=2, output_tokens=1),
)
assert telemetry_env.callbacks._metric_state.active_count() == 0
assert telemetry_env.callbacks._span_state.active_count() == 0
@pytest.mark.asyncio
@pytest.mark.parametrize(
"control_error",
[asyncio.CancelledError(), KeyboardInterrupt(), SystemExit()],
)
async def test_llm_first_chunk_control_failure_cleans_state_before_propagating(
telemetry_env: SimpleNamespace,
monkeypatch: pytest.MonkeyPatch,
control_error: BaseException,
) -> None:
await telemetry_env.framework.trigger(
LLMCallEvents.LLM_STREAM_INPUT,
[{"role": "user", "content": "stream"}],
model="control-stream-model",
)
original_span_key = telemetry_env.callbacks._span_key
def fail_span_key(span):
del span
raise control_error
monkeypatch.setattr(telemetry_env.callbacks, "_span_key", fail_span_key)
chunk = SimpleNamespace(content="first")
with pytest.raises(type(control_error)):
await telemetry_env.callbacks._on_llm_stream_output(result=chunk)
assert telemetry_env.callbacks._metric_state.active_count() == 0
assert telemetry_env.callbacks._span_state.active_count() == 0
monkeypatch.setattr(telemetry_env.callbacks, "_span_key", original_span_key)
await telemetry_env.framework.trigger(
LLMCallEvents.LLM_OUTPUT,
is_stream=True,
response="done",
)
@pytest.mark.asyncio
async def test_tool_output_enrichment_failure_preserves_business_result_and_cleans_state(
telemetry_env: SimpleNamespace,
monkeypatch: pytest.MonkeyPatch,
) -> None:
await telemetry_env.framework.trigger(
ToolCallEvents.TOOL_CALL_STARTED,
tool_name="fragile_tool",
tool_id="fragile-call",
inputs={"value": 1},
)
result = {"business": "value"}
warnings: list[str] = []
original_bounded_json = telemetry_env.callbacks._bounded_json
def fail_serialization(value, *, redact):
del value, redact
raise RuntimeError("serialization failed")
monkeypatch.setattr(telemetry_env.callbacks, "_bounded_json", fail_serialization)
monkeypatch.setattr(
callbacks_module._LOGGER,
"warning",
lambda message, **kwargs: warnings.append(message),
)
returned = await telemetry_env.callbacks._on_tool_output(
tool_name="fragile_tool",
tool_id="fragile-call",
result=result,
)
assert returned is result
assert warnings == ["Tool telemetry output enrichment failed"]
assert telemetry_env.callbacks._metric_state.active_count() == 0
assert telemetry_env.callbacks._span_state.active_count() == 0
assert len(_metric_points(telemetry_env.reader, "gen_ai.tool.call.count")) == 1
assert len(_metric_points(telemetry_env.reader, "gen_ai.tool.duration")) == 1
monkeypatch.setattr(
telemetry_env.callbacks,
"_bounded_json",
original_bounded_json,
)
await telemetry_env.framework.trigger(
ToolCallEvents.TOOL_CALL_FINISHED,
tool_name="fragile_tool",
tool_id="fragile-call",
result=result,
)
assert len(_metric_points(telemetry_env.reader, "gen_ai.tool.call.count")) == 1
assert len(_metric_points(telemetry_env.reader, "gen_ai.tool.duration")) == 1
@pytest.mark.asyncio
async def test_tool_skill_failure_preserves_result_and_settles_basic_metrics_once(
telemetry_env: SimpleNamespace,
monkeypatch: pytest.MonkeyPatch,
) -> None:
inputs = {"skill_name": "fragile-skill", "skill_id": "skill-1"}
await telemetry_env.framework.trigger(
ToolCallEvents.TOOL_CALL_STARTED,
tool_name="skill_tool",
tool_id="fragile-skill-call",
inputs=inputs,
)
original_extract_skill = callbacks_module.extract_skill
def fail_skill(*args, **kwargs):
del args, kwargs
raise RuntimeError("skill extraction failed")
warnings: list[str] = []
monkeypatch.setattr(callbacks_module, "extract_skill", fail_skill)
monkeypatch.setattr(
callbacks_module._LOGGER,
"warning",
lambda message, **kwargs: warnings.append(message),
)
result = {"success": True}
returned = await telemetry_env.callbacks._on_tool_output(
tool_name="skill_tool",
tool_id="fragile-skill-call",
inputs=inputs,
result=result,
)
assert returned is result
assert warnings == ["Tool telemetry output enrichment failed"]
assert telemetry_env.callbacks._metric_state.active_count() == 0
assert telemetry_env.callbacks._span_state.active_count() == 0
assert len(_metric_points(telemetry_env.reader, "gen_ai.tool.call.count")) == 1
assert len(_metric_points(telemetry_env.reader, "gen_ai.tool.duration")) == 1
monkeypatch.setattr(callbacks_module, "extract_skill", original_extract_skill)
await telemetry_env.framework.trigger(
ToolCallEvents.TOOL_CALL_FINISHED,
tool_name="skill_tool",
tool_id="fragile-skill-call",
inputs=inputs,
result=result,
)
assert len(_metric_points(telemetry_env.reader, "gen_ai.tool.call.count")) == 1
assert len(_metric_points(telemetry_env.reader, "gen_ai.tool.duration")) == 1
@pytest.mark.asyncio
async def test_llm_output_enrichment_failure_preserves_business_result_and_cleans_state(
telemetry_env: SimpleNamespace,
monkeypatch: pytest.MonkeyPatch,
) -> None:
await telemetry_env.framework.trigger(
LLMCallEvents.LLM_INVOKE_INPUT,
[{"role": "user", "content": "hello"}],
model="fragile-model",
)
warnings: list[str] = []
original_serializer = callbacks_module.serialize_output_message
def fail_serialization(*args, **kwargs):
del args, kwargs
raise RuntimeError("serialization failed")
monkeypatch.setattr(
callbacks_module, "serialize_output_message", fail_serialization
)
monkeypatch.setattr(
callbacks_module._LOGGER,
"warning",
lambda message, **kwargs: warnings.append(message),
)
result = SimpleNamespace(
content="business result",
tool_calls=[],
usage_metadata=SimpleNamespace(input_tokens=7, output_tokens=3),
)
returned = await telemetry_env.callbacks._on_llm_invoke_output(result=result)
assert returned is result
assert warnings == ["LLM telemetry output enrichment failed"]
assert telemetry_env.callbacks._metric_state.active_count() == 0
assert telemetry_env.callbacks._span_state.active_count() == 0
assert (
len(_metric_points(telemetry_env.reader, "gen_ai.client.operation.count")) == 1
)
assert (
len(_metric_points(telemetry_env.reader, "gen_ai.client.operation.duration"))
== 1
)
token_points = {
point.attributes["gen_ai.token.type"]: point.value
for point in _metric_points(
telemetry_env.reader,
"gen_ai.client.token.usage",
)
}
assert token_points == {"input": 7, "output": 3}
monkeypatch.setattr(
callbacks_module,
"serialize_output_message",
original_serializer,
)
await telemetry_env.framework.trigger(
LLMCallEvents.LLM_INVOKE_OUTPUT,
result=result,
)
assert (
len(_metric_points(telemetry_env.reader, "gen_ai.client.operation.count")) == 1
)
@pytest.mark.asyncio
async def test_agent_output_enrichment_failure_preserves_business_result_and_cleans_state(
telemetry_env: SimpleNamespace,
monkeypatch: pytest.MonkeyPatch,
) -> None:
await telemetry_env.callbacks._on_agent_input({"query": "hello"})
warnings: list[str] = []
def fail_find(*args, **kwargs):
del args, kwargs
raise RuntimeError("span lookup failed")
monkeypatch.setattr(telemetry_env.registry, "find", fail_find)
monkeypatch.setattr(
callbacks_module._LOGGER,
"warning",
lambda message, **kwargs: warnings.append(message),
)
result = object()
returned = await telemetry_env.callbacks._on_agent_invoke_output(result=result)
assert returned is result
assert warnings == ["Agent telemetry output enrichment failed"]
assert telemetry_env.callbacks._metric_state.active_count() == 0
assert telemetry_env.callbacks._span_state.active_count() == 0
assert len(_metric_points(telemetry_env.reader, "jiuwenclaw.agent.duration")) == 1
@pytest.mark.asyncio
async def test_tool_error_and_empty_ids_cleanup_without_cross_call_reuse(
telemetry_env: SimpleNamespace,
) -> None:
for tool_name in ("first", "second"):
await telemetry_env.framework.trigger(
ToolCallEvents.TOOL_CALL_STARTED,
tool_name=tool_name,
tool_id="",
inputs={"name": tool_name},
)
await telemetry_env.framework.trigger(
ToolCallEvents.TOOL_CALL_ERROR,
tool_name=tool_name,
tool_id="",
error=TimeoutError(tool_name),
)
assert telemetry_env.callbacks._span_state.active_count() == 0
assert telemetry_env.callbacks._metric_state.active_count() == 0
assert len(_metric_points(telemetry_env.reader, "gen_ai.tool.call.count")) == 2
assert len(_metric_points(telemetry_env.reader, "gen_ai.tool.error.count")) == 2
telemetry_env.provider.force_flush()
spans = [
item
for item in telemetry_env.exporter.get_finished_spans()
if item.name in {"tool.first", "tool.second"}
]
assert {item.attributes["error.type"] for item in spans} == {"TimeoutError"}
assert all(item.attributes["jiuwenclaw.timeout"] is True for item in spans)
@pytest.mark.asyncio
async def test_tool_error_enrichment_uses_agentcore_context_without_otel_attachment(
telemetry_env: SimpleNamespace,
) -> None:
invalid_context_token = otel_context.attach(
trace.set_span_in_context(trace.INVALID_SPAN)
)
try:
await telemetry_env.framework.trigger(
ToolCallEvents.TOOL_CALL_STARTED,
tool_name="detached",
tool_id="detached-call",
inputs={"value": 1},
)
await telemetry_env.framework.trigger(
ToolCallEvents.TOOL_CALL_ERROR,
tool_name="detached",
tool_id="detached-call",
error=TimeoutError("detached"),
)
finally:
otel_context.detach(invalid_context_token)
telemetry_env.provider.force_flush()
span = next(
item
for item in telemetry_env.exporter.get_finished_spans()
if item.name == "tool.detached"
)
assert span.attributes["error.type"] == "TimeoutError"
assert span.attributes["jiuwenclaw.timeout"] is True
assert telemetry_env.callbacks._span_state.active_count() == 0
assert telemetry_env.callbacks._metric_state.active_count() == 0
@pytest.mark.asyncio
@pytest.mark.parametrize("timeout_seconds", [0.01, 0, -1])
async def test_token_count_timeout_only_drops_context_breakdown(
monkeypatch: pytest.MonkeyPatch,
timeout_seconds: float,
) -> None:
def slow_count(messages, tools):
del messages, tools
time.sleep(0.2)
raise AssertionError("cancelled work must not affect callback")
monkeypatch.setattr(
"jiuwenswarm.telemetry.enrichment.callbacks.count_context_tokens",
slow_count,
)
metrics = _RecordingMetrics()
callbacks = RichTelemetryCallbacks(
span_registry=SpanRegistryProcessor(max_spans=8, ttl_seconds=60),
metrics=metrics,
config=SimpleNamespace(token_count_timeout_seconds=timeout_seconds),
)
framework = AsyncCallbackFramework()
await callbacks.register(framework)
started_at = time.monotonic()
try:
await framework.trigger(
LLMCallEvents.LLM_INVOKE_INPUT,
messages=[{"role": "user", "content": "quick"}],
model="timeout-model",
)
result = object()
returned = await framework.trigger(
LLMCallEvents.LLM_INVOKE_OUTPUT, result=result
)
finally:
await callbacks.unregister(framework)
assert time.monotonic() - started_at < 0.15
assert returned[0] is result
assert callbacks._metric_state.active_count() == 0
assert {call[1] for call in metrics.calls} >= {
"gen_ai.client.operation.count",
"gen_ai.client.operation.duration",
}
@pytest.mark.asyncio
@pytest.mark.parametrize(
"control_error",
[asyncio.CancelledError(), KeyboardInterrupt(), SystemExit()],
)
async def test_control_metric_errors_propagate_after_state_cleanup(
control_error: BaseException,
) -> None:
callbacks = RichTelemetryCallbacks(
span_registry=SpanRegistryProcessor(max_spans=8, ttl_seconds=60),
metrics=_RecordingMetrics(control_error),
config=TelemetryConfig(enabled=True),
)
framework = AsyncCallbackFramework()
await callbacks.register(framework)
try:
await framework.trigger(
LLMCallEvents.LLM_INVOKE_INPUT,
messages=[],
model="control-model",
)
with pytest.raises(type(control_error)):
await framework.trigger(
LLMCallEvents.LLM_INVOKE_OUTPUT,
result=SimpleNamespace(content="answer", tool_calls=[]),
)
assert callbacks._metric_state.active_count() == 0
finally:
await callbacks.unregister(framework)
@pytest.mark.asyncio
async def test_register_rolls_back_partial_failure_and_unregister_is_retryable() -> (
None
):
callbacks = RichTelemetryCallbacks(
span_registry=SpanRegistryProcessor(max_spans=8, ttl_seconds=60),
metrics=_Metrics(),
config=TelemetryConfig(),
)
failing_register = _FailingFramework(fail_register_at=4)
with pytest.raises(RuntimeError, match="register failed"):
await callbacks.register(failing_register)
assert callbacks._registered == []
assert callbacks._framework is None
assert not failing_register.list_events(namespace=NAMESPACE)
framework = _FailingFramework()
await callbacks.register(framework)
assert callbacks._framework is framework
framework.fail_unregister_callback = callbacks._registered[-1][1]
with pytest.raises(RuntimeError, match="unregister failed"):
await callbacks.unregister(framework)
assert len(callbacks._registered) == 1
assert callbacks._framework is framework
await callbacks.unregister(framework)
assert callbacks._registered == []
assert callbacks._framework is None
assert not framework.list_events(namespace=NAMESPACE)
rollback_failure = _FailingFramework(
fail_register_at=4,
fail_unregister_at=1,
)
with pytest.raises(RuntimeError, match="register failed"):
await callbacks.register(rollback_failure)
assert len(callbacks._registered) == 1
assert callbacks._framework is rollback_failure
assert len(rollback_failure.list_events(namespace=NAMESPACE)) == 1
await callbacks.unregister(rollback_failure)
assert callbacks._registered == []
assert callbacks._framework is None
assert not rollback_failure.list_events(namespace=NAMESPACE)
@pytest.mark.asyncio
async def test_unregister_rejects_a_different_framework_without_losing_state() -> None:
callbacks = RichTelemetryCallbacks(
span_registry=SpanRegistryProcessor(max_spans=8, ttl_seconds=60),
metrics=_Metrics(),
config=TelemetryConfig(),
)
registered_framework = AsyncCallbackFramework()
other_framework = AsyncCallbackFramework()
await callbacks.register(registered_framework)
registered = list(callbacks._registered)
with pytest.raises(ValueError, match="different callback framework"):
await callbacks.unregister(other_framework)
assert callbacks._registered == registered
assert callbacks._framework is registered_framework
assert len(registered_framework.list_events(namespace=NAMESPACE)) == 13
assert not other_framework.list_events(namespace=NAMESPACE)
await callbacks.unregister(registered_framework)
assert callbacks._registered == []
assert callbacks._framework is None
assert not registered_framework.list_events(namespace=NAMESPACE)
@pytest.mark.asyncio
async def test_registration_is_idempotent_and_core_ordering_is_verified(
telemetry_env: SimpleNamespace,
) -> None:
registered = list(telemetry_env.callbacks._registered)
await telemetry_env.callbacks.register(telemetry_env.framework)
assert telemetry_env.callbacks._registered == registered
assert telemetry_env.callbacks.validate_core_ordering(telemetry_env.framework)
await telemetry_env.callbacks.unregister(telemetry_env.framework)
assert not telemetry_env.callbacks.validate_core_ordering(telemetry_env.framework)
await telemetry_env.callbacks.unregister(telemetry_env.framework)
@pytest.mark.asyncio
async def test_context_token_metrics_are_emitted_without_a_recording_span(
monkeypatch: pytest.MonkeyPatch,
request_metric_channel: None,
) -> None:
del request_metric_channel
monkeypatch.setattr(
callbacks_module,
"count_context_tokens",
lambda messages, tools: ContextTokenBreakdown(
tool_definitions=14,
per_tool_tokens=(("search", 4), ("weather", 7)),
skill=12,
per_skill_tokens=(("research", 5), ("weather", 7)),
),
)
metrics = _RecordingMetrics()
callbacks = RichTelemetryCallbacks(
span_registry=SpanRegistryProcessor(max_spans=8, ttl_seconds=60),
metrics=metrics,
config=TelemetryConfig(enabled=True),
)
framework = AsyncCallbackFramework()
await callbacks.register(framework)
try:
await framework.trigger(
LLMCallEvents.LLM_INVOKE_INPUT,
messages=[],
tools=[{"function": {"name": "search"}}],
model="token-model",
)
await framework.trigger(
LLMCallEvents.LLM_INVOKE_OUTPUT,
result=SimpleNamespace(content="ok", tool_calls=[]),
)
finally:
await callbacks.unregister(framework)
tool_points = [
call for call in metrics.calls if call[1] == "gen_ai.tool.token.usage"
]
skill_points = [
call for call in metrics.calls if call[1] == "gen_ai.skill.token.usage"
]
assert [(call[2], call[3]) for call in tool_points] == [
(
4,
{
"gen_ai.tool.name": "search",
"gen_ai.request.model": "token-model",
"jiuwenclaw.channel.id": "web",
},
),
(
7,
{
"gen_ai.tool.name": "weather",
"gen_ai.request.model": "token-model",
"jiuwenclaw.channel.id": "web",
},
),
]
assert {(call[2], frozenset(call[3].items())) for call in skill_points} == {
(
5,
frozenset(
{
"gen_ai.skill.name": "research",
"gen_ai.request.model": "token-model",
"jiuwenclaw.channel.id": "web",
}.items()
),
),
(
7,
frozenset(
{
"gen_ai.skill.name": "weather",
"gen_ai.request.model": "token-model",
"jiuwenclaw.channel.id": "web",
}.items()
),
),
}
@pytest.mark.asyncio
async def test_token_count_completion_does_not_swallow_external_cancel(telemetry_env, monkeypatch):
loop = asyncio.get_running_loop()
finished = loop.create_future()
started = asyncio.Event()
def pending_executor(*args):
started.set()
return finished
monkeypatch.setattr(loop, "run_in_executor", pending_executor)
baseline = telemetry_env.registry.active_count()
task = asyncio.create_task(telemetry_env.framework.trigger(
LLMCallEvents.LLM_INVOKE_INPUT, messages=[], model="cancel-race-model",
))
async with asyncio.timeout(5):
await started.wait()
finished.set_result(ContextTokenBreakdown())
assert task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
assert telemetry_env.callbacks._metric_state.active_count() == 0
assert telemetry_env.callbacks._span_state.active_count() == 0
assert telemetry_env.registry.active_count() == baseline
@pytest.mark.parametrize(
"control_error",
[asyncio.CancelledError(), KeyboardInterrupt(), SystemExit()],
)
def test_input_control_errors_propagate_after_state_cleanup(
monkeypatch: pytest.MonkeyPatch,
control_error: BaseException,
) -> None:
async def _inline_to_thread(func, /, *args, **kwargs):
return func(*args, **kwargs)
async def _inline_wait_for(aw, timeout=None):
del timeout
return await aw
# KeyboardInterrupt/SystemExit inside pytest-asyncio (or asyncio.wait_for's
# nested task) aborts the whole pytest session. Drive the coroutine with
# an explicitly closed event loop so pytest.raises can catch them without
# leaving its selector sockets behind after a prior async test.
monkeypatch.setattr(callbacks_module.asyncio, "to_thread", _inline_to_thread)
monkeypatch.setattr(callbacks_module.asyncio, "wait_for", _inline_wait_for)
def raise_control(messages, tools):
del messages, tools
raise control_error
monkeypatch.setattr(callbacks_module, "count_context_tokens", raise_control)
callbacks = RichTelemetryCallbacks(
span_registry=SpanRegistryProcessor(max_spans=8, ttl_seconds=60),
metrics=_Metrics(),
config=TelemetryConfig(enabled=True),
)
async def _exercise() -> None:
await callbacks._on_llm_invoke_input(messages=[], model="control-model")
def _run_in_closed_loop() -> None:
loop = asyncio.new_event_loop()
try:
loop.run_until_complete(_exercise())
finally:
loop.run_until_complete(loop.shutdown_asyncgens())
loop.close()
with pytest.raises(type(control_error)):
_run_in_closed_loop()
assert callbacks._metric_state.active_count() == 0
assert callbacks._span_state.active_count() == 0
@pytest.mark.asyncio
@pytest.mark.parametrize(
"interceptor_error",
[asyncio.CancelledError("intercepted"), KeyboardInterrupt("intercepted")],
)
async def test_real_framework_cancellation_closes_core_llm_span_and_metrics_once(
telemetry_env: SimpleNamespace,
monkeypatch: pytest.MonkeyPatch,
interceptor_error: BaseException,
) -> None:
# Exercise cancellation, not the independent token-count timeout. Under CI
# load the 0.5s timeout can finish the task before the waiter calls cancel().
monkeypatch.setattr(
telemetry_env.callbacks,
"_config",
SimpleNamespace(
**vars(telemetry_env.callbacks._config),
token_count_timeout_seconds=float("inf"),
),
)
token_count_started = threading.Event()
release_token_count = threading.Event()
def slow_token_count(messages, tools):
del messages, tools
token_count_started.set()
# Only the finally block releases this worker; it must not complete
# naturally before the cancellation assertion on a slow runner.
release_token_count.wait()
return ContextTokenBreakdown()
monkeypatch.setattr(callbacks_module, "count_context_tokens", slow_token_count)
third_party_calls: list[BaseException] = []
async def intercept_error(**kwargs):
third_party_calls.append(kwargs["error"])
raise interceptor_error
telemetry_env.framework.register_sync(
LLMCallEvents.LLM_CALL_ERROR,
intercept_error,
priority=50,
namespace="test.control.interceptor",
)
baseline_spans = telemetry_env.registry.active_count()
task = asyncio.create_task(
telemetry_env.framework.trigger(
LLMCallEvents.LLM_INVOKE_INPUT,
messages=[{"role": "user", "content": "cancel me"}],
model="slow-token-model",
)
)
try:
assert await asyncio.to_thread(token_count_started.wait, 1)
assert task.cancel(), "token-count task must still be pending"
with pytest.raises(asyncio.CancelledError):
await task
finally:
release_token_count.set()
telemetry_env.framework.unregister_sync(
LLMCallEvents.LLM_CALL_ERROR,
intercept_error,
)
telemetry_env.provider.force_flush()
assert third_party_calls == []
assert telemetry_env.callbacks._metric_state.active_count() == 0
assert telemetry_env.callbacks._span_state.active_count() == 0
assert telemetry_env.registry.active_count() == baseline_spans
assert (
telemetry_env.registry.find_latest(
telemetry_env.parent.get_span_context().trace_id,
name="llm.call",
)
is None
)
spans = [
span
for span in telemetry_env.exporter.get_finished_spans()
if span.name == "llm.call"
]
assert len(spans) == 1
assert spans[0].status.status_code.name == "ERROR"
operation_counts = _metric_points(
telemetry_env.reader,
"gen_ai.client.operation.count",
)
assert len(operation_counts) == 1
assert operation_counts[0].value == 1
@pytest.mark.asyncio
async def test_ordinary_enrichment_error_degrades_without_breaking_business_call(
telemetry_env: SimpleNamespace,
monkeypatch: pytest.MonkeyPatch,
) -> None:
warnings: list[str] = []
monkeypatch.setattr(
callbacks_module,
"serialize_input_messages",
lambda *args, **kwargs: (_ for _ in ()).throw(RuntimeError("telemetry failed")),
)
monkeypatch.setattr(
callbacks_module._LOGGER,
"warning",
lambda message, **kwargs: warnings.append(message),
)
await telemetry_env.framework.trigger(
LLMCallEvents.LLM_INVOKE_INPUT,
messages=[{"role": "user", "content": "hello"}],
model="degraded-model",
)
assert telemetry_env.callbacks._metric_state.active_count() == 1
assert telemetry_env.callbacks._span_state.active_count() == 1
result = object()
returned = await telemetry_env.framework.trigger(
LLMCallEvents.LLM_INVOKE_OUTPUT,
result=result,
)
assert returned[0] is result
assert warnings == ["LLM telemetry input enrichment failed"]
assert telemetry_env.callbacks._metric_state.active_count() == 0
assert telemetry_env.callbacks._span_state.active_count() == 0
assert (
len(_metric_points(telemetry_env.reader, "gen_ai.client.operation.count")) == 1
)
assert (
len(_metric_points(telemetry_env.reader, "gen_ai.client.operation.duration"))
== 1
)
@pytest.mark.asyncio
async def test_state_registries_enforce_capacity_ttl_and_remove_indexes() -> None:
now = time.monotonic()
span_states = EnrichmentStateRegistry(max_entries=2, ttl_seconds=1)
for sequence in range(1, 4):
span_states.put(
(sequence, sequence),
callbacks_module._CallState(started_at=now, sequence=sequence),
)
assert span_states.active_count() == 2
assert span_states.get((1, 1)) is None
assert span_states.prune(now=now + 1) == 2
assert span_states.active_count() == 0
metric_states = MetricCallStateRegistry(max_entries=2, ttl_seconds=1)
for identity in ("one", "two", "three"):
metric_states.start(
"llm",
identity,
callbacks_module._CallState(started_at=now),
)
assert metric_states.active_count() == 2
assert metric_states.finish("llm", "one") is None
assert metric_states.prune(now=now + 1) == 2
assert metric_states.active_count() == 0
assert metric_states.finish("llm", "two") is None
expired_span_states = EnrichmentStateRegistry(max_entries=1, ttl_seconds=1)
expired_span_states.put(
(9, 9),
callbacks_module._CallState(started_at=now - 2, sequence=1),
)
assert expired_span_states.get((9, 9)) is None
expired_metric_states = MetricCallStateRegistry(max_entries=1, ttl_seconds=1)
expired_metric_states.start(
"llm",
"expired",
callbacks_module._CallState(started_at=now - 2),
)
assert expired_metric_states.finish("llm", "expired") is None
skill_sessions = SkillSessionRegistry(max_entries=2, ttl_seconds=1)
replacement = callbacks_module._SkillSession(
started_at=now,
version="replacement",
channel_id="web",
)
skill_sessions.start(
1,
"session-1",
"one",
callbacks_module._SkillSession(now - 0.5, "old", "cli"),
)
skill_sessions.start(1, "session-1", "one", replacement)
assert skill_sessions.active_count() == 1
assert skill_sessions.finish(1, "session-1", "one") == replacement
for index in range(3):
skill_sessions.start(
index,
f"session-{index}",
f"skill-{index}",
callbacks_module._SkillSession(now + index / 10, str(index), "web"),
)
assert skill_sessions.active_count() == 2
assert skill_sessions.finish(0, "session-0", "skill-0") is None
assert skill_sessions.prune(now=now + 2) == 2
assert skill_sessions.active_count() == 0
@pytest.mark.parametrize(
"registry_type",
[EnrichmentStateRegistry, MetricCallStateRegistry, SkillSessionRegistry],
)
def test_state_registries_reject_invalid_bounds(registry_type) -> None:
with pytest.raises(ValueError, match="max_entries"):
registry_type(max_entries=0, ttl_seconds=1)
with pytest.raises(ValueError, match="ttl_seconds"):
registry_type(max_entries=1, ttl_seconds=0)
@pytest.mark.asyncio
async def test_skill_sessions_isolate_invalid_trace_by_session_across_tasks() -> None:
registry = SkillSessionRegistry(max_entries=4, ttl_seconds=60)
gate = asyncio.Event()
ready = 0
async def worker(session_id: str, version: str):
nonlocal ready
session = callbacks_module._SkillSession(
started_at=time.monotonic(),
version=version,
channel_id="web",
)
registry.start(0, session_id, "forecast", session)
ready += 1
if ready == 2:
gate.set()
await gate.wait()
return registry.finish(0, session_id, "forecast")
first, second = await asyncio.gather(
worker("session-a", "v1"),
worker("session-b", "v2"),
)
assert (first.version, second.version) == ("v1", "v2")
assert registry.active_count() == 0
@pytest.mark.asyncio
async def test_empty_id_concurrent_calls_are_isolated_by_task() -> None:
metrics = _RecordingMetrics()
callbacks = RichTelemetryCallbacks(
span_registry=SpanRegistryProcessor(max_spans=8, ttl_seconds=60),
metrics=metrics,
config=TelemetryConfig(enabled=True),
)
framework = AsyncCallbackFramework()
await callbacks.register(framework)
ready: list[str] = []
gate = asyncio.Event()
async def worker(model: str, tool_name: str) -> None:
await framework.trigger(
LLMCallEvents.LLM_INVOKE_INPUT,
messages=[],
model=model,
)
await framework.trigger(
ToolCallEvents.TOOL_CALL_STARTED,
tool_name=tool_name,
tool_id="",
inputs={},
)
ready.append(model)
if len(ready) == 2:
gate.set()
await gate.wait()
await framework.trigger(
ToolCallEvents.TOOL_CALL_FINISHED,
tool_name=tool_name,
tool_id="",
result={"ok": True},
)
await framework.trigger(
LLMCallEvents.LLM_INVOKE_OUTPUT,
result=SimpleNamespace(content="ok", tool_calls=[]),
)
try:
await asyncio.gather(
worker("model-a", "tool-a"),
worker("model-b", "tool-b"),
)
finally:
await callbacks.unregister(framework)
operation_counts = [
call for call in metrics.calls if call[1] == "gen_ai.client.operation.count"
]
tool_counts = [
call for call in metrics.calls if call[1] == "gen_ai.tool.call.count"
]
assert {call[3]["gen_ai.request.model"] for call in operation_counts} == {
"model-a",
"model-b",
}
assert {call[3]["gen_ai.tool.name"] for call in tool_counts} == {
"tool-a",
"tool-b",
}
assert callbacks._metric_state.active_count() == 0
@pytest.mark.asyncio
async def test_same_tool_id_concurrent_calls_keep_state_and_spans_task_local(
telemetry_env: SimpleNamespace,
) -> None:
first_started = asyncio.Event()
second_started = asyncio.Event()
first_finished = asyncio.Event()
async def first_call() -> None:
await telemetry_env.framework.trigger(
ToolCallEvents.TOOL_CALL_STARTED,
tool_name="shared",
tool_id="static-card-id",
inputs={"owner": "first"},
)
first_started.set()
await second_started.wait()
await telemetry_env.framework.trigger(
ToolCallEvents.TOOL_CALL_FINISHED,
tool_name="shared",
tool_id="static-card-id",
result={"owner": "first"},
)
first_finished.set()
async def second_call() -> None:
await first_started.wait()
await telemetry_env.framework.trigger(
ToolCallEvents.TOOL_CALL_STARTED,
tool_name="shared",
tool_id="static-card-id",
inputs={"owner": "second"},
)
second_started.set()
await first_finished.wait()
await telemetry_env.framework.trigger(
ToolCallEvents.TOOL_CALL_FINISHED,
tool_name="shared",
tool_id="static-card-id",
result={"owner": "second"},
)
await asyncio.gather(first_call(), second_call())
telemetry_env.provider.force_flush()
spans = [
span
for span in telemetry_env.exporter.get_finished_spans()
if span.name == "tool.shared"
]
assert len(spans) == 2
observed_pairs = {
(
span.attributes["gen_ai.tool.arguments"],
span.attributes["gen_ai.tool.result"],
)
for span in spans
}
assert observed_pairs == {
('{"owner": "first"}', '{"owner": "first"}'),
('{"owner": "second"}', '{"owner": "second"}'),
}
assert telemetry_env.callbacks._metric_state.active_count() == 0
assert telemetry_env.callbacks._span_state.active_count() == 0