Skip to content

JiuwenSwarm is an intelligent AI Agent built on openJiuwen. It extends the powerful capabilities of large language models directly to your fingertips through various communication apps you use daily.

Python
15.7 KB
Raw Blame
"""Concurrent request isolation and terminal cleanup integration tests."""

from __future__ import annotations

import asyncio
from collections.abc import AsyncIterator
from types import SimpleNamespace

import pytest
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import InMemoryMetricReader
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
    InMemorySpanExporter,
)

from openjiuwen.agent_teams.observability import (
    ObservabilityConfig,
    init_observability,
    shutdown_observability,
)
from openjiuwen.core.runner import Runner
from openjiuwen.core.runner.callback import LLMCallEvents, ToolCallEvents

from jiuwenswarm.extensions.identity_provider import IdentityInfo, IdentityStore
from jiuwenswarm.telemetry.config import TelemetryConfig
from jiuwenswarm.telemetry.enrichment.callbacks import RichTelemetryCallbacks
from jiuwenswarm.telemetry.metrics import TelemetryMetrics
from jiuwenswarm.telemetry.request_context import TraceBindingRegistry
from jiuwenswarm.telemetry.span_registry import SpanRegistryProcessor

# CI Python 3.11: asyncio.Runner.close() can raise during pytest-asyncio
# teardown; pytest.ini ``filterwarnings = error`` would turn that into ERROR.
pytestmark = pytest.mark.filterwarnings(
    "ignore:An exception occurred during teardown of an asyncio.Runner:RuntimeWarning"
)


def _release_asyncio_default_executor() -> None:
    """Drop the loop default executor so pytest-asyncio Runner.close() cannot hang.

    OTel provider shutdown / callback unregister may leave work on the default
    ThreadPoolExecutor; ``asyncio.Runner.close`` then blocks in
    ``shutdown_default_executor`` until pytest-timeout kills the teardown.
    """
    try:
        loop = asyncio.get_running_loop()
    except RuntimeError:
        return
    executor = getattr(loop, "_default_executor", None)
    if executor is None:
        return
    # Python 3.11 rejects set_default_executor(None); clear the private slot
    # so Runner.close() skips shutdown_default_executor on a hung pool.
    loop._default_executor = None
    executor.shutdown(wait=False, cancel_futures=True)


@pytest.fixture
async def concurrent_env() -> AsyncIterator[SimpleNamespace]:
    shutdown_observability()
    identity_token = IdentityStore.set_identity(None)
    exporter = InMemorySpanExporter()
    provider = TracerProvider(
        resource=Resource.create({"service.name": "concurrent-integration"})
    )
    span_registry = SpanRegistryProcessor(max_spans=256, ttl_seconds=60)
    provider.add_span_processor(span_registry)
    provider.add_span_processor(SimpleSpanProcessor(exporter))
    metric_reader = InMemoryMetricReader()
    meter_provider = MeterProvider(metric_readers=[metric_reader])
    callbacks = RichTelemetryCallbacks(
        span_registry=span_registry,
        metrics=TelemetryMetrics(meter_provider),
        config=TelemetryConfig(enabled=True),
    )
    init_observability(
        ObservabilityConfig(enabled=True, backend="otlp"),
        tracer_provider_override=provider,
        owns_provider=False,
    )
    await callbacks.register(Runner.callback_framework)
    runtime = SimpleNamespace(
        is_unified_active=lambda: True,
        tracer_provider=provider,
        span_registry=span_registry,
        trace_bindings=TraceBindingRegistry(max_bindings=64, ttl_seconds=60),
    )
    try:
        yield SimpleNamespace(
            callbacks=callbacks,
            exporter=exporter,
            framework=Runner.callback_framework,
            metric_reader=metric_reader,
            meter_provider=meter_provider,
            provider=provider,
            runtime=runtime,
            span_registry=span_registry,
        )
    finally:
        try:
            await callbacks.unregister(Runner.callback_framework)
            shutdown_observability()
            provider.force_flush(timeout_millis=5_000)
            meter_provider.force_flush(timeout_millis=5_000)
            meter_provider.shutdown(timeout_millis=5_000)
            provider.shutdown()
        finally:
            IdentityStore.clear(identity_token)
            _release_asyncio_default_executor()
            await asyncio.sleep(0)


def _metric_points(reader: InMemoryMetricReader, name: str) -> list[object]:
    data = reader.get_metrics_data()
    if data is None:
        return []
    return [
        point
        for resource_metrics in data.resource_metrics
        for scope_metrics in resource_metrics.scope_metrics
        for metric in scope_metrics.metrics
        if metric.name == name
        for point in metric.data.data_points
    ]


@pytest.mark.asyncio
async def test_two_sessions_keep_identity_metrics_and_cancellation_isolated(
    concurrent_env: SimpleNamespace,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from jiuwenswarm.agents.harness import agent_observability

    monkeypatch.setattr(
        agent_observability,
        "_get_unified_runtime",
        lambda: concurrent_env.runtime,
    )
    started = {name: asyncio.Event() for name in ("a", "b")}
    release_b = asyncio.Event()
    handles: dict[str, object] = {}
    results: dict[str, object] = {}
    warmup_usage = {"a": (3, 1), "b": (7, 2)}

    async def _run_session(name: str) -> object:
        session_id = f"session-{name}"
        request_id = f"request-{name}"
        call_id = f"llm-{name}"
        token = IdentityStore.set_identity(
            IdentityInfo(
                user_id=f"user-{name}",
                domain_id=f"domain-{name}",
                app_id=f"app-{name}",
            )
        )
        handle = agent_observability.open_agent_run_span(
            session_id=session_id,
            request_id=request_id,
            channel_id=f"channel-{name}",
            mode="code.normal",
        )
        assert handle is not None
        handles[name] = handle
        try:
            warmup_call_id = f"warmup-{name}"
            await concurrent_env.framework.trigger(
                LLMCallEvents.LLM_INVOKE_INPUT,
                messages=[{"role": "user", "content": f"warmup-{name}"}],
                model=f"warmup-model-{name}",
                call_id=warmup_call_id,
            )
            warmup_input, warmup_output = warmup_usage[name]
            warmup_result = SimpleNamespace(
                content=f"warmup-result-{name}",
                finish_reason="stop",
                tool_calls=[],
                usage_metadata=SimpleNamespace(
                    input_tokens=warmup_input,
                    output_tokens=warmup_output,
                ),
            )
            await concurrent_env.framework.trigger(
                LLMCallEvents.LLM_INVOKE_OUTPUT,
                result=warmup_result,
                call_id=warmup_call_id,
            )
            await concurrent_env.framework.trigger(
                LLMCallEvents.LLM_INVOKE_INPUT,
                messages=[{"role": "user", "content": f"prompt-{name}"}],
                model=f"active-model-{name}",
                call_id=call_id,
            )
            started[name].set()
            if name == "a":
                try:
                    await asyncio.Event().wait()
                except asyncio.CancelledError as error:
                    await concurrent_env.framework.trigger(
                        LLMCallEvents.LLM_CALL_ERROR,
                        error=error,
                        call_id=call_id,
                    )
                    raise
            await release_b.wait()
            result = SimpleNamespace(
                content=f"result-{name}",
                finish_reason="stop",
                tool_calls=[],
                usage_metadata=SimpleNamespace(
                    input_tokens=11,
                    output_tokens=4,
                ),
            )
            callbacks = await concurrent_env.framework.trigger(
                LLMCallEvents.LLM_INVOKE_OUTPUT,
                result=result,
                call_id=call_id,
            )
            await concurrent_env.framework.trigger(
                ToolCallEvents.TOOL_CALL_STARTED,
                tool_name="tool-b",
                tool_id="tool-b",
                inputs=(("input-b",), {}),
            )
            await concurrent_env.framework.trigger(
                ToolCallEvents.TOOL_CALL_FINISHED,
                tool_name="tool-b",
                tool_id="tool-b",
                inputs=(("input-b",), {}),
                result={"session": name},
            )
            results[name] = result
            assert callbacks[0] is result
            return result
        finally:
            agent_observability.close_agent_run_span(
                handle,
                session_id=session_id,
            )
            IdentityStore.clear(token)

    task_a = asyncio.create_task(_run_session("a"))
    task_b = asyncio.create_task(_run_session("b"))
    await asyncio.wait_for(
        asyncio.gather(started["a"].wait(), started["b"].wait()),
        timeout=2.0,
    )
    task_a.cancel()
    outcome_a = (await asyncio.gather(task_a, return_exceptions=True))[0]
    assert isinstance(outcome_a, asyncio.CancelledError)
    try:
        assert not task_b.done()
        root_b = handles["b"].root_span
        assert root_b.is_recording()
        live_b = concurrent_env.runtime.trace_bindings.resolve(
            "session-b",
            "request-b",
        )
        assert live_b is not None
        assert live_b.root_span is root_b
    finally:
        release_b.set()
        outcome_b = await task_b

    assert outcome_b is results["b"]
    concurrent_env.provider.force_flush()
    spans = list(concurrent_env.exporter.get_finished_spans())
    root_a = handles["a"].root_span
    trace_a = root_a.get_span_context().trace_id
    trace_b = root_b.get_span_context().trace_id
    assert trace_a != trace_b
    spans_a = [span for span in spans if span.context.trace_id == trace_a]
    spans_b = [span for span in spans if span.context.trace_id == trace_b]

    assert {span.attributes.get("user.id") for span in spans_a} == {"user-a"}
    assert {span.attributes.get("user.id") for span in spans_b} == {"user-b"}
    assert [span.name for span in spans_a].count("llm.call") == 2
    assert [span.name for span in spans_b].count("llm.call") == 2
    assert "tool.tool-b" not in [span.name for span in spans_a]
    assert "tool.tool-b" in [span.name for span in spans_b]
    assert {span.attributes.get("jiuwenclaw.request.id") for span in spans_a} == {
        "request-a"
    }
    assert {span.attributes.get("jiuwenclaw.request.id") for span in spans_b} == {
        "request-b"
    }
    cancelled_span = next(
        span
        for span in spans_a
        if span.name == "llm.call" and span.attributes.get("jiuwenclaw.canceled")
    )
    assert cancelled_span.attributes["jiuwenclaw.canceled"] is True
    usage_a = {
        (
            span.attributes.get("gen_ai.usage.input_tokens"),
            span.attributes.get("gen_ai.usage.output_tokens"),
        )
        for span in spans_a
        if span.name == "llm.call" and "gen_ai.usage.input_tokens" in span.attributes
    }
    usage_b = {
        (
            span.attributes.get("gen_ai.usage.input_tokens"),
            span.attributes.get("gen_ai.usage.output_tokens"),
        )
        for span in spans_b
        if span.name == "llm.call" and "gen_ai.usage.input_tokens" in span.attributes
    }
    assert usage_a == {(3, 1)}
    assert usage_b == {(7, 2), (11, 4)}

    operation_points = _metric_points(
        concurrent_env.metric_reader,
        "gen_ai.client.operation.count",
    )
    assert {point.attributes["gen_ai.request.model"] for point in operation_points} == {
        "warmup-model-a",
        "warmup-model-b",
        "active-model-a",
        "active-model-b",
    }
    token_points = _metric_points(
        concurrent_env.metric_reader,
        "gen_ai.client.token.usage",
    )
    token_values = {
        (
            point.attributes["gen_ai.request.model"],
            point.attributes["gen_ai.token.type"],
        ): point.value
        for point in token_points
    }
    assert token_values == {
        ("warmup-model-a", "input"): 3,
        ("warmup-model-a", "output"): 1,
        ("warmup-model-b", "input"): 7,
        ("warmup-model-b", "output"): 2,
        ("active-model-b", "input"): 11,
        ("active-model-b", "output"): 4,
    }
    assert (
        concurrent_env.runtime.trace_bindings.resolve("session-a", "request-a") is None
    )
    assert (
        concurrent_env.runtime.trace_bindings.resolve("session-b", "request-b") is None
    )
    assert concurrent_env.span_registry.active_count() == 0
    assert concurrent_env.callbacks._span_state.active_count() == 0
    assert concurrent_env.callbacks._metric_state.active_count() == 0


@pytest.mark.asyncio
@pytest.mark.parametrize(
    "business_error",
    [ValueError("business failed"), TimeoutError("model timed out")],
    ids=["error", "timeout"],
)
async def test_error_and_timeout_preserve_original_exception_and_cleanup(
    concurrent_env: SimpleNamespace,
    monkeypatch: pytest.MonkeyPatch,
    business_error: Exception,
) -> None:
    from jiuwenswarm.agents.harness import agent_observability

    monkeypatch.setattr(
        agent_observability,
        "_get_unified_runtime",
        lambda: concurrent_env.runtime,
    )
    handle = agent_observability.open_agent_run_span(
        session_id="session-error",
        request_id="request-error",
        channel_id="channel-error",
        mode="agent.plan",
    )
    assert handle is not None
    is_tool_timeout = isinstance(business_error, TimeoutError)

    async def _run_failing_business_operation() -> None:
        if is_tool_timeout:
            await concurrent_env.framework.trigger(
                ToolCallEvents.TOOL_CALL_STARTED,
                tool_name="slow_tool",
                tool_id="timeout-tool",
                inputs=(("wait",), {}),
            )
        else:
            await concurrent_env.framework.trigger(
                LLMCallEvents.LLM_INVOKE_INPUT,
                messages=[{"role": "user", "content": "fail"}],
                model="error-model",
                call_id="error-call",
            )
        try:
            raise business_error
        except BaseException as error:
            if is_tool_timeout:
                await concurrent_env.framework.trigger(
                    ToolCallEvents.TOOL_CALL_ERROR,
                    error=error,
                    tool_name="slow_tool",
                    tool_id="timeout-tool",
                )
            else:
                await concurrent_env.framework.trigger(
                    LLMCallEvents.LLM_CALL_ERROR,
                    error=error,
                    call_id="error-call",
                )
            raise

    try:
        with pytest.raises(type(business_error)) as raised:
            await _run_failing_business_operation()
    finally:
        agent_observability.close_agent_run_span(
            handle,
            session_id="session-error",
        )

    assert raised.value is business_error
    concurrent_env.provider.force_flush()
    operation_span = next(
        span
        for span in concurrent_env.exporter.get_finished_spans()
        if span.name == ("tool.slow_tool" if is_tool_timeout else "llm.call")
    )
    assert operation_span.attributes["error.type"] == type(business_error).__name__
    if is_tool_timeout:
        assert operation_span.attributes["jiuwenclaw.timeout"] is True
    assert (
        concurrent_env.runtime.trace_bindings.resolve("session-error", "request-error")
        is None
    )
    assert concurrent_env.span_registry.active_count() == 0
    assert concurrent_env.callbacks._span_state.active_count() == 0
    assert concurrent_env.callbacks._metric_state.active_count() == 0