AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
feat(backend): give copilot turns and graph node runs their own Sentry traces (#15462)
Nicholas Tindle2ef655354d6f
diff --git a/autogpt_platform/backend/backend/api/features/chat/routes.py b/autogpt_platform/backend/backend/api/features/chat/routes.pyindex 680098da33..02dd581b5d 100644--- a/autogpt_platform/backend/backend/api/features/chat/routes.py+++ b/autogpt_platform/backend/backend/api/features/chat/routes.py@@ -156,6 +156,7 @@ from backend.copilot.transports import (is_deployment_chat_available,save_default_chat_route,)+from backend.copilot.turn_tracing import MODEL_OUTPUT_CHUNKSfrom backend.data.credit import UsageTransactionMetadata, get_user_credit_modelfrom backend.data.redis_client import get_redis_asyncfrom backend.data.understanding import get_business_understanding@@ -164,6 +165,7 @@ from backend.data.workspace_folder import resolve_attachable_workspace_foldersfrom backend.integrations.codex.access import enforce_codex_access_httpfrom backend.integrations.credentials_store import provider_matchesfrom backend.integrations.creds_manager import IntegrationCredentialsManager+from backend.util import tracingfrom backend.util.background import spawn_background_taskfrom backend.util.exceptions import InsufficientBalanceError, NotFoundErrorfrom backend.util.feature_flag import Flag, is_feature_enabled@@ -2064,6 +2066,8 @@ async def stream_chat_post(# Per-turn stream is always fresh (unique turn_id), subscribe from beginningsubscribe_from_id = "0-0"+ # This request's transaction, which the relay below records its latency on.+ stream_transaction = tracing.current_transaction()# SSE endpoint that subscribes to the task's streamasync def event_generator() -> AsyncGenerator[str, None]:@@ -2077,6 +2081,7 @@ async def stream_chat_post()subscriber_queue = Nonefirst_chunk_yielded = False+ first_token_yielded = Falsechunks_yielded = 0try:# Subscribe from the position we captured before enqueuing@@ -2116,6 +2121,20 @@ async def stream_chat_post(}},)+ tracing.set_timing(+ "copilot.stream.first_chunk_ms",+ (time_module.perf_counter() - stream_start_time) * 1000,+ span=stream_transaction,+ )+ if not first_token_yielded and isinstance(+ chunk, MODEL_OUTPUT_CHUNKS+ ):+ first_token_yielded = True+ tracing.set_timing(+ "copilot.stream.ttft_ms",+ (time_module.perf_counter() - stream_start_time) * 1000,+ span=stream_transaction,+ )yield stream_registry.sse_frame(entry)@@ -2187,6 +2206,9 @@ async def stream_chat_post(}},)+ tracing.set_data(+ "copilot.stream.chunk_count", chunks_yielded, span=stream_transaction+ )yield "data: [DONE]\n\n"return StreamingResponse(diff --git a/autogpt_platform/backend/backend/api/features/chat/routes_test.py b/autogpt_platform/backend/backend/api/features/chat/routes_test.pyindex 50a71c6a4c..f9f3ed6a6f 100644--- a/autogpt_platform/backend/backend/api/features/chat/routes_test.py+++ b/autogpt_platform/backend/backend/api/features/chat/routes_test.py@@ -1,5 +1,6 @@"""Tests for chat API routes: session title update, file attachment validation, usage, and rate limiting."""+import asynciofrom datetime import UTC, datetime, timedeltafrom types import SimpleNamespacefrom typing import get_args@@ -10,6 +11,7 @@ import fastapi.testclientimport pytestimport pytest_mockfrom pydantic import SecretStr+from sentry_sdk.integrations.asgi import SentryAsgiMiddlewarefrom backend.api.features.chat import routes as chat_routesfrom backend.api.features.chat.routes import _strip_injected_context@@ -18,6 +20,7 @@ from backend.copilot.config import CopilotLlmAuthProviderfrom backend.copilot.model import ChatSessionfrom backend.copilot.offers import EntitlementUnavailablefrom backend.copilot.rate_limit import SubscriptionTier+from backend.copilot.response_model import StreamFinish, StreamStatus, StreamTextDeltafrom backend.copilot.tools.models import ExpertSoulUpdatedResponsefrom backend.data.model import OAuth2Credentialsfrom backend.data.workspace_scope import WorkspaceScope@@ -352,6 +355,52 @@ def _mock_stream_internals()+def test_stream_chat_records_when_the_first_chunk_and_token_reached_the_client(+ mocker: pytest_mock.MockerFixture, sentry_transactions+) -> None:+ """The relay's latency lands on the request's own transaction, which the+ ASGI middleware starts for it in production."""+ _mock_stream_internals(mocker)+ relayed: asyncio.Queue = asyncio.Queue()+ for chunk in (+ StreamStatus(message="Message received…"),+ StreamTextDelta(id="t", delta="hi"),+ StreamFinish(),+ ):+ relayed.put_nowait((None, chunk))+ mocker.patch.object(+ chat_routes.stream_registry,+ "subscribe_to_session",+ new_callable=AsyncMock,+ return_value=relayed,+ )+ mocker.patch.object(+ chat_routes.stream_registry, "unsubscribe_from_session", new_callable=AsyncMock+ )+ mocker.patch.object(+ chat_routes.stream_registry, "publish_chunk", new_callable=AsyncMock+ )+ mocker.patch(+ "backend.api.features.chat.routes.get_global_rate_limits",+ new_callable=AsyncMock,+ return_value=(0, 0, SubscriptionTier.BASIC),+ )+ mocker.patch(+ "backend.api.features.chat.routes.check_rate_limit", new_callable=AsyncMock+ )++ response = fastapi.testclient.TestClient(SentryAsgiMiddleware(app)).post(+ "/sessions/sess-1/stream", json={"message": "hello"}+ )++ assert response.status_code == 200+ [request] = sentry_transactions+ data = request["contexts"]["trace"]["data"]+ assert request["contexts"]["trace"]["op"] == "http.server"+ assert 0 <= data["copilot.stream.first_chunk_ms"] <= data["copilot.stream.ttft_ms"]+ assert data["copilot.stream.chunk_count"] == 3++def test_stream_chat_accepts_20_file_ids(mocker: pytest_mock.MockerFixture):"""Exactly 20 file_ids should be accepted (not rejected by validation)."""_mock_stream_internals(mocker)diff --git a/autogpt_platform/backend/backend/conftest.py b/autogpt_platform/backend/backend/conftest.pyindex fcfe17caec..281ed6949a 100644--- a/autogpt_platform/backend/backend/conftest.py+++ b/autogpt_platform/backend/backend/conftest.py@@ -2,6 +2,7 @@ import inspectimport loggingimport osimport sys+from collections.abc import Iteratorfrom contextlib import asynccontextmanagerfrom datetime import datetime, timezonefrom multiprocessing import get_all_start_methods, get_context, set_forkserver_preload@@ -11,13 +12,17 @@ from uuid import uuid4import pytestimport pytest_asyncio+import sentry_sdkfrom dotenv import load_dotenv+from sentry_sdk.envelope import Envelope+from sentry_sdk.transport import Transportfrom backend.data import versioned_cachefrom backend.data.subscription_lookup_cache import ACTIVE_SUBSCRIPTIONSfrom backend.data.subscription_lookup_unconfirmed import UnconfirmedInvalidationsfrom backend.data.subscription_pending_cache import PENDING_CHANGESfrom backend.util.logging import configure_logging+from backend.util.tracing import traces_samplerload_dotenv()@@ -171,6 +176,42 @@ def target_user_id() -> str:return "5e53486c-cf57-477e-ba2a-cb02dc828e1c"+class _SentryTransactionCapture(Transport):+ """Keeps the transactions a test's Sentry client would have sent."""++ def __init__(self) -> None:+ super().__init__()+ self.transactions: list[dict] = []++ def capture_envelope(self, envelope: Envelope) -> None:+ for item in envelope.items:+ if item.type == "transaction" and item.payload.json is not None:+ self.transactions.append(item.payload.json)+++def sentry_transactions() -> Iterator[list[dict]]:+ """A live Sentry client for the test, sampling as production does. Every+ transaction it finishes is appended, as sent, to the yielded list."""+ capture = _SentryTransactionCapture()+ client = sentry_sdk.Client(+ dsn="https://[email protected]/1",+ transport=capture,+ traces_sampler=traces_sampler,+ default_integrations=False,+ auto_enabling_integrations=False,+ send_client_reports=False,+ )+ global_scope = sentry_sdk.get_global_scope()+ previous = global_scope.client+ global_scope.set_client(client)+ try:+ yield capture.transactions+ finally:+ global_scope.set_client(previous)+ client.close()++async def _create_user_with_loop_retry(user_data: dict) -> None:"""Create a user, retrying once on a transient ``Event loop is closed``.diff --git a/autogpt_platform/backend/backend/copilot/executor/processor.py b/autogpt_platform/backend/backend/copilot/executor/processor.pyindex 20a61ba895..1d01cd47ac 100644--- a/autogpt_platform/backend/backend/copilot/executor/processor.py+++ b/autogpt_platform/backend/backend/copilot/executor/processor.py@@ -31,6 +31,7 @@ from backend.copilot.sdk.dummy import stream_chat_completion_dummyfrom backend.copilot.stream_heartbeat import wrap_stream_with_heartbeatfrom backend.copilot.tools.agent_browser import close_browser_daemonfrom backend.copilot.trial_cost_context import trial_cost_context+from backend.copilot.turn_tracing import TurnTrace, turn_transactionfrom backend.data.model import OAuth2Credentialsfrom backend.executor.cluster_lock import ClusterLockfrom backend.integrations.codex.transport import CodexCredentialIntegrityError@@ -540,7 +541,8 @@ class CoPilotProcessor:cluster_lock: ClusterLock,log: CoPilotLogMetadata,):- """Async execution logic for a CoPilot turn.+ """Async execution logic for a CoPilot turn, run as one Sentry+ ``copilot.turn`` transaction.Calls the chat completion service (SDK or baseline) and publishesresults to the stream registry.@@ -551,6 +553,17 @@ class CoPilotProcessor:cluster_lock: Distributed lock for refreshlog: Structured logger"""+ with turn_transaction(entry) as turn:+ await self._run_turn(entry, cancel, cluster_lock, log, turn)++ async def _run_turn(+ self,+ entry: CoPilotExecutionEntry,+ cancel: threading.Event,+ cluster_lock: ClusterLock,+ log: CoPilotLogMetadata,+ turn: TurnTrace,+ ):last_refresh = time.monotonic()refresh_interval = 30.0 # Refresh lock every 30 secondserror_msg = None@@ -638,6 +651,7 @@ class CoPilotProcessor:credential_lease = Noneraise RuntimeError("codex_credential_not_found") from Nonestream_fn = sdk_service.stream_chat_completion_sdk+ engine = "sdk"log.info("Using Claude SDK with Codex subscription transport")elif entry.llm_auth_provider == "microsoft_365_copilot":if entry.user_id is None:@@ -677,6 +691,7 @@ class CoPilotProcessor:)raise RuntimeError(code) from errorstream_fn = cast(Callable, stream_chat_completion_microsoft_365)+ engine = "microsoft_365_copilot"log.info("Using Microsoft 365 Copilot Chat API transport")else:if entry.llm_credential_id is not None:@@ -687,6 +702,7 @@ class CoPilotProcessor:if config.test_mode:stream_fn = stream_chat_completion_dummy+ engine = "dummy"log.warning("Using DUMMY service (CHAT_TEST_MODE=true)")else:use_sdk = await resolve_use_sdk(@@ -712,6 +728,7 @@ class CoPilotProcessor:if use_sdkelse stream_chat_completion_baseline)+ engine = "sdk" if use_sdk else "baseline"log.info(f"Using {'SDK' if use_sdk else 'baseline'} service")await cost_context_stack.enter_async_context(@@ -726,6 +743,7 @@ class CoPilotProcessor:envelope = taint_for_source_platform(entry.envelope, session)set_turn_unattended(session, scheduled=entry.unattended)set_turn_credential_pins(entry.credential_pins)+ turn.engine_started(engine)raw_stream = stream_fn(session_id=entry.session_id,message=entry.message or None,@@ -765,6 +783,7 @@ class CoPilotProcessor:# until GC eventually runs.try:async for chunk in published_stream:+ turn.observe(chunk)if cancel.is_set():log.info("Cancel requested, breaking stream")break@@ -815,6 +834,7 @@ class CoPilotProcessor:log.error(f"Failed to release chat credential: {release_err}")finally:# After the release, which can still fail the turn.+ turn.set_outcome(error_msg, stream_registry.CANCELLED_MESSAGE)if scheduled_watch is not None:scheduled_watch.report(error_msg)try:diff --git a/autogpt_platform/backend/backend/copilot/executor/processor_test.py b/autogpt_platform/backend/backend/copilot/executor/processor_test.pyindex 84190dd45b..dd2610123a 100644--- a/autogpt_platform/backend/backend/copilot/executor/processor_test.py+++ b/autogpt_platform/backend/backend/copilot/executor/processor_test.py@@ -11,6 +11,7 @@ import asyncioimport concurrent.futuresimport loggingimport threading+import timefrom contextlib import asynccontextmanagerfrom types import SimpleNamespacefrom unittest.mock import AsyncMock, MagicMock, patch@@ -34,6 +35,12 @@ from backend.copilot.expert_context import ()from backend.copilot.model import ChatSessionfrom backend.copilot.rate_limit import UserPaywalledError+from backend.copilot.response_model import (+ StreamError,+ StreamStatus,+ StreamTextDelta,+ StreamToolInputAvailable,+)from backend.data.model import OAuth2Credentialsfrom backend.integrations.codex.transport import (CodexCredentialBusyError,@@ -322,6 +329,86 @@ class TestExecuteAsyncAclose:assert published.aclose_called is True+ @pytest.mark.asyncio+ async def test_a_turn_is_one_transaction_in_the_dispatchers_trace(+ self, sentry_transactions+ ) -> None:+ trace_id, parent_span_id = "a" * 32, "b" * 16+ entry = _make_entry().model_copy(+ update={+ "trace_headers": {"sentry-trace": f"{trace_id}-{parent_span_id}-1"},+ "enqueued_at": time.time() - 2,+ }+ )+ published = _TrackedStream(+ events=[+ StreamStatus(message="Thinking…"),+ StreamTextDelta(id="t", delta="hi"),+ StreamToolInputAvailable(toolCallId="c", toolName="run_block"),+ ]+ )++ patches = self._patches(published)+ with patches[0], patches[1], patches[2], patches[3], patches[4]:+ await CoPilotProcessor()._execute_async(+ entry, threading.Event(), MagicMock(), _make_log()+ )++ [turn] = sentry_transactions+ trace = turn["contexts"]["trace"]+ assert turn["transaction"] == "copilot.turn"+ assert trace["op"] == "copilot.turn"+ assert trace["trace_id"] == trace_id+ assert trace["parent_span_id"] == parent_span_id+ assert trace["status"] == "ok"+ assert turn["tags"] == {+ "copilot.llm_auth_provider": "platform",+ "copilot.model_tier": "default",+ "copilot.turn.origin": "chat",+ "copilot.engine": "dummy",+ "copilot.turn.outcome": "completed",+ }+ data = trace["data"]+ assert data["copilot.session_id"] == "sess-1"+ assert data["copilot.turn_id"] == "turn-1"+ assert data["copilot.turn.queue_wait_ms"] >= 2000+ assert 0 <= data["copilot.turn.setup_ms"] <= data["copilot.turn.ttft_ms"]+ assert data["copilot.turn.ttft_ms"] <= data["copilot.turn.duration_ms"]+ assert data["copilot.turn.chunk_count"] == 3+ assert data["copilot.turn.tool_call_count"] == 1++ @pytest.mark.asyncio+ async def test_a_turn_that_streams_an_error_is_failed(+ self, sentry_transactions+ ) -> None:+ published = _TrackedStream(events=[StreamError(errorText="provider down")])++ patches = self._patches(published)+ with patches[0], patches[1], patches[2], patches[3], patches[4]:+ await CoPilotProcessor()._execute_async(+ _make_entry(), threading.Event(), MagicMock(), _make_log()+ )++ [turn] = sentry_transactions+ assert turn["tags"]["copilot.turn.outcome"] == "failed"+ assert turn["contexts"]["trace"]["status"] == "internal_error"+ assert "copilot.turn.ttft_ms" not in turn["contexts"]["trace"]["data"]++ @pytest.mark.asyncio+ async def test_a_cancelled_turn_is_cancelled(self, sentry_transactions) -> None:+ cancel = threading.Event()+ cancel.set()++ patches = self._patches(_TrackedStream(events=[MagicMock()]))+ with patches[0], patches[1], patches[2], patches[3], patches[4]:+ await CoPilotProcessor()._execute_async(+ _make_entry(), cancel, MagicMock(), _make_log()+ )++ [turn] = sentry_transactions+ assert turn["tags"]["copilot.turn.outcome"] == "cancelled"+ assert turn["contexts"]["trace"]["status"] == "cancelled"+@pytest.mark.asyncioasync def test_persisted_expert_session_is_passed_to_engine(self) -> None:published = _TrackedStream(events=[])diff --git a/autogpt_platform/backend/backend/copilot/executor/utils.py b/autogpt_platform/backend/backend/copilot/executor/utils.pyindex e52c90d627..4cb63f67d2 100644--- a/autogpt_platform/backend/backend/copilot/executor/utils.py+++ b/autogpt_platform/backend/backend/copilot/executor/utils.py@@ -7,6 +7,8 @@"""import logging+import time+from collections.abc import Mappingfrom typing import TYPE_CHECKING, Anyfrom uuid import uuid4@@ -36,6 +38,7 @@ from backend.copilot.tree import (root_envelope,)from backend.data.rabbitmq import Exchange, ExchangeType, Queue, RabbitMQConfig+from backend.util import tracingfrom backend.util.logging import TruncatedLogger, is_structured_logging_enabledfrom backend.util.settings import Config@@ -352,6 +355,14 @@ class CoPilotExecutionEntry(BaseModel):"""Set when the scheduler fired this turn. ``None`` for every othercaller, and for entries queued before the field existed."""+ trace_headers: dict[str, str] = Field(default_factory=dict)+ """``sentry-trace``/``baggage`` of the transaction that dispatched the turn,+ which the turn's own transaction continues. Empty when the dispatcher was+ not traced or is not the turn's cause, e.g. a promotion from the queue."""++ enqueued_at: float = 0.0+ """Unix-epoch seconds when the turn was published, for its queue wait."""+class CancelCoPilotEvent(BaseModel):"""Event to cancel a CoPilot operation."""@@ -384,6 +395,7 @@ async def enqueue_copilot_turn(unattended: bool = False,credential_pins: CredentialPins | None = None,scheduled: ScheduledTurnOrigin | None = None,+ trace_headers: Mapping[str, str] | None = None,) -> None:"""Enqueue a CoPilot task for processing by the executor service.@@ -395,6 +407,9 @@ async def enqueue_copilot_turn(``CoPilotExecutionEntry.envelope`` stays optional so a message enqueued byan older worker still decodes.+ The turn continues the Sentry trace in ``trace_headers``, by default that+ of the transaction this is called in.+Args:session_id: Chat session ID (also used for dedup/locking)user_id: User ID (may be None for anonymous users)@@ -430,6 +445,12 @@ async def enqueue_copilot_turn(unattended=unattended,credential_pins=credential_pins or {},scheduled=scheduled,+ trace_headers=(+ dict(trace_headers)+ if trace_headers is not None+ else tracing.trace_propagation_headers()+ ),+ enqueued_at=time.time(),)queue_client = await get_async_copilot_queue()@@ -590,12 +611,18 @@ async def dispatch_turn(``envelope`` was admitted before (an approval wake carries the one itscall was held under): it is re-checked instead, narrowed by today's``permissions``, and its node is not released if dispatch fails.++ The same rule decides the turn's Sentry trace: it continues the caller's+ only when it derives its envelope from the caller's context, because a+ root or a re-admitted turn was caused by something other than the code+ dispatching it."""# Local import: stream_registry imports executor.utils (the# COPILOT_CONSUMER_TIMEOUT_SECONDS constant) → top-level circular.from backend.copilot import stream_registryadmitted_here = envelope is None+ trace_headers = None if admitted_here and not root else {}if envelope is None:envelope = await _admitted_turn_envelope(turn_id, session_id, user_id, permissions, spawn, root=root@@ -649,6 +676,7 @@ async def dispatch_turn(unattended=unattended,credential_pins=credential_pins,scheduled=scheduled,+ trace_headers=trace_headers,)slot.keep()committed = Truediff --git a/autogpt_platform/backend/backend/copilot/executor/utils_tracing_test.py b/autogpt_platform/backend/backend/copilot/executor/utils_tracing_test.pynew file mode 100644index 0000000000..ee707555cb--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/executor/utils_tracing_test.py@@ -0,0 +1,112 @@+"""A turn's Sentry trace is handed over in its queue message, from the+transaction that caused the turn to the turn's own transaction."""++import time+from unittest.mock import AsyncMock, MagicMock, patch++import pytest++from backend.copilot.executor import utils+from backend.copilot.executor.utils import CoPilotExecutionEntry+from backend.copilot.tree import root_envelope+from backend.util import tracing+++def _published_entry(publish: AsyncMock) -> CoPilotExecutionEntry:+ return CoPilotExecutionEntry.model_validate_json(+ publish.await_args.kwargs["message"]+ )+++def copilot_queue():+ queue = MagicMock(publish_message=AsyncMock())+ with patch(+ "backend.util.clients.get_async_copilot_queue",+ new=AsyncMock(return_value=queue),+ ):+ yield queue+++async def _enqueue(**kwargs) -> None:+ await utils.enqueue_copilot_turn(+ session_id="s1",+ user_id="u1",+ message="hi",+ turn_id="t1",+ envelope=root_envelope("t1"),+ **kwargs,+ )+++async def test_a_turn_enqueued_inside_a_transaction_continues_its_trace(+ sentry_transactions, copilot_queue+) -> None:+ before = time.time()+ with tracing.start_transaction("http.server", "/stream") as request:+ await _enqueue()++ entry = _published_entry(copilot_queue.publish_message)+ assert entry.trace_headers["sentry-trace"].startswith(request.trace_id)+ assert "baggage" in entry.trace_headers+ assert before <= entry.enqueued_at <= time.time()+++async def test_an_untraced_enqueue_hands_over_no_trace(+ sentry_transactions, copilot_queue+) -> None:+ await _enqueue()++ assert _published_entry(copilot_queue.publish_message).trace_headers == {}+++def test_an_entry_from_before_the_trace_fields_still_decodes() -> None:+ legacy = '{"session_id": "s1", "user_id": "u1", "message": "hi"}'++ entry = CoPilotExecutionEntry.model_validate_json(legacy)++ assert entry.trace_headers == {}+ assert entry.enqueued_at == 0.0+++ "dispatch_kwargs, continues_caller",+ [+ ({}, True),+ ({"root": True}, False),+ ({"envelope": root_envelope("held")}, False),+ ],+ ids=["derived-from-caller", "minted-root", "re-admitted"],+)+async def test_dispatch_continues_only_the_trace_of_the_turns_cause(+ sentry_transactions, dispatch_kwargs: dict, continues_caller: bool+) -> None:+ """A promoted or woken turn is dispatched from inside whatever turn just+ ended, which did not cause it, so only a turn whose envelope is derived+ from the caller joins the caller's trace."""+ enqueue = AsyncMock()+ with (+ patch.object(+ utils,+ "_admitted_turn_envelope",+ new=AsyncMock(return_value=root_envelope("t1")),+ ),+ patch.object(utils, "readmit_turn", new=AsyncMock()),+ patch("backend.copilot.stream_registry.create_session", new=AsyncMock()),+ patch.object(utils, "enqueue_copilot_turn", new=enqueue),+ ):+ with tracing.start_transaction("copilot.turn", "copilot.turn"):+ await utils.dispatch_turn(+ MagicMock(),+ session_id="s1",+ user_id="u1",+ turn_id="t1",+ message="hi",+ **dispatch_kwargs,+ )++ trace_headers = enqueue.await_args.kwargs["trace_headers"]+ assert trace_headers == (None if continues_caller else {})diff --git a/autogpt_platform/backend/backend/copilot/stream_registry.py b/autogpt_platform/backend/backend/copilot/stream_registry.pyindex 0e3fa71041..236c7b0b80 100644--- a/autogpt_platform/backend/backend/copilot/stream_registry.py+++ b/autogpt_platform/backend/backend/copilot/stream_registry.py@@ -17,7 +17,7 @@ Subscribers:import asyncioimport loggingimport time-from collections.abc import AsyncGenerator, Iterable, Mapping+from collections.abc import AsyncGenerator, Coroutine, Iterable, Mappingfrom dataclasses import dataclass, fieldfrom datetime import datetime, timezonefrom typing import Any, Literal@@ -36,6 +36,7 @@ from backend.data.notification_bus import ()from backend.data.redis_client import get_redis_asyncfrom backend.data.redis_helpers import as_str, hash_compare_and_set+from backend.util import tracingfrom .config import ChatConfigfrom .constants import STREAM_LOCK_PREFIX@@ -351,6 +352,10 @@ async def publish_chunk(All delivery is via Redis Streams - no in-memory state.+ A turn publishes every chunk it streams, so these Redis calls are kept+ off the turn's Sentry transaction, which would otherwise outgrow the+ SDK's span limit.+Args:turn_id: Turn ID (per-turn UUID) identifying the streamchunk: The stream response chunk to publish@@ -361,6 +366,16 @@ async def publish_chunk(Returns:The Redis Stream message ID"""+ with tracing.suppress_child_spans():+ return await _publish_chunk(turn_id, chunk, session_id=session_id)+++async def _publish_chunk(+ turn_id: str,+ chunk: StreamBaseResponse,+ *,+ session_id: str | None,+) -> str:start_time = time.perf_counter()chunk_type = type(chunk).__name__chunk_json = chunk.model_dump_json()@@ -686,11 +701,21 @@ def _start_listener(turn_id: str,) -> None:listener_task = asyncio.create_task(- _stream_listener(session_id, queue, last_id, log_meta, turn_id)+ _without_child_spans(+ _stream_listener(session_id, queue, last_id, log_meta, turn_id)+ ))_listener_sessions[id(queue)] = (session_id, listener_task)+async def _without_child_spans(listener: Coroutine[Any, Any, None]) -> None:+ """Run a listener off its subscriber's Sentry transaction. It polls Redis+ for as long as the turn streams, which would outgrow the SDK's span limit+ on the ``/stream`` request relaying a long turn."""+ with tracing.suppress_child_spans():+ await listener++def _chunk_from_fields(fields: dict[str, str]) -> StreamBaseResponse | None:if "data" not in fields:return Nonediff --git a/autogpt_platform/backend/backend/copilot/stream_registry_tracing_test.py b/autogpt_platform/backend/backend/copilot/stream_registry_tracing_test.pynew file mode 100644index 0000000000..f83a1f1e58--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/stream_registry_tracing_test.py@@ -0,0 +1,103 @@+"""Per-chunk and per-poll Redis calls stay off Sentry transactions, so a long+turn's transaction and the ``/stream`` request relaying it stay bounded."""++import asyncio+from unittest.mock import AsyncMock, patch++import pytest+import sentry_sdk++from backend.copilot import stream_registry+from backend.copilot.response_model import StreamFinish, StreamTextDelta+from backend.util import tracing++[email protected](autouse=True)+def _clear_listener_sessions():+ stream_registry._listener_sessions.clear()+ yield+ stream_registry._listener_sessions.clear()+++class _SpanPerCommandRedis:+ """Redis with Sentry's integration on: every command is a ``db.redis`` span+ named after it. Serves a running turn whose live tail is ``live``."""++ def __init__(self, live: list[tuple[str, dict[str, str]]] | None = None):+ self._live = live or []++ async def _command(self, name: str, result):+ with sentry_sdk.start_span(op="db.redis", name=name):+ return result++ async def hgetall(self, _key):+ meta = {+ "user_id": "u1",+ "session_id": "sess-1",+ "turn_id": "turn-1",+ "status": "running",+ }+ return await self._command("HGETALL", meta)++ async def xread(self, _streams, count=None, block=None):+ if block is None:+ return await self._command("XREAD", [])+ live, self._live = self._live, []+ return await self._command("XREAD BLOCK", [("stream", live)] if live else [])++ async def hmget(self, _key, _fields):+ return await self._command("HMGET", ["completed", "turn-1"])++ async def xadd(self, _key, _fields):+ return await self._command("XADD", "1-0")++ async def expire(self, _key, _ttl):+ return await self._command("EXPIRE", True)+++def _stored(chunk) -> dict[str, str]:+ return {"data": chunk.model_dump_json()}+++async def test_the_relay_keeps_its_polls_off_the_request_transaction(+ sentry_transactions,+):+ redis = _SpanPerCommandRedis(+ live=[+ ("1-0", _stored(StreamTextDelta(id="t", delta="hi"))),+ ("1-1", _stored(StreamFinish())),+ ]+ )+ with patch.object(+ stream_registry, "get_redis_async", new=AsyncMock(return_value=redis)+ ):+ with tracing.start_transaction("http.server", "/stream"):+ queue = await stream_registry.subscribe_to_session("sess-1", "u1")+ assert queue is not None+ delivered = [await asyncio.wait_for(queue.get(), 5) for _ in range(2)]++ assert [type(chunk).__name__ for _, chunk in delivered] == [+ "StreamTextDelta",+ "StreamFinish",+ ]+ [request] = sentry_transactions+ assert [span["description"] for span in request["spans"]] == ["HGETALL", "XREAD"]+++async def test_publishing_a_chunk_adds_no_spans_to_the_turn(sentry_transactions):+ with patch.object(+ stream_registry,+ "get_redis_async",+ new=AsyncMock(return_value=_SpanPerCommandRedis()),+ ):+ with tracing.start_transaction("copilot.turn", "copilot.turn"):+ for _ in range(3):+ message_id = await stream_registry.publish_chunk(+ "turn-1", StreamTextDelta(id="t", delta="x"), session_id="s"+ )++ assert message_id == "1-0"+ [turn] = sentry_transactions+ assert turn["spans"] == []diff --git a/autogpt_platform/backend/backend/copilot/tools/base.py b/autogpt_platform/backend/backend/copilot/tools/base.pyindex 22b259ba67..d908dfeb63 100644--- a/autogpt_platform/backend/backend/copilot/tools/base.py+++ b/autogpt_platform/backend/backend/copilot/tools/base.py@@ -10,8 +10,10 @@ from openai.types.chat import ChatCompletionToolParamfrom backend.copilot.context import get_current_envelopefrom backend.copilot.model import ChatSessionfrom backend.copilot.response_model import StreamToolOutputAvailable+from backend.copilot.turn_tracing import tool_transactionfrom backend.data.activity_event import ActivityEventDraftfrom backend.data.db_accessors import activity_event_db, workspace_db+from backend.util import tracingfrom backend.util.truncate import truncatefrom backend.util.workspace import WorkspaceManager@@ -410,6 +412,9 @@ class BaseTool:) -> StreamToolOutputAvailable:"""Execute the tool with authentication check.+ Each call is its own Sentry ``copilot.tool`` transaction, named after+ the tool, in the trace of the turn that made it.+Args:user_id: User ID (None for anonymous users)session_id: Chat session ID@@ -419,6 +424,20 @@ class BaseTool:Pydantic response object"""+ with tool_transaction(self.name, tool_call_id, session.session_id) as span:+ output = await self._execute_checked(+ user_id, session, tool_call_id, **kwargs+ )+ span.set_tag("copilot.tool.success", "true" if output.success else "false")+ return output++ async def _execute_checked(+ self,+ user_id: str | None,+ session: ChatSession,+ tool_call_id: str,+ **kwargs,+ ) -> StreamToolOutputAvailable:kwargs.pop(GATE_APPROVED, None)if self.requires_auth and not user_id:logger.warning(@@ -518,6 +537,7 @@ class BaseTool:)except Exception as e:logger.warning("Error in %s", self.name, exc_info=True)+ tracing.set_status("internal_error")return StreamToolOutputAvailable(toolCallId=tool_call_id,toolName=self.name,diff --git a/autogpt_platform/backend/backend/copilot/tools/base_tracing_test.py b/autogpt_platform/backend/backend/copilot/tools/base_tracing_test.pynew file mode 100644index 0000000000..3b017339c3--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/tools/base_tracing_test.py@@ -0,0 +1,65 @@+"""Every tool call is its own Sentry transaction in its turn's trace, so a+turn's transaction stays bounded however many tools the turn runs."""++from unittest.mock import MagicMock++import pytest+import sentry_sdk++from backend.copilot.tools.base import BaseTool+from backend.copilot.tools.models import ErrorResponse+from backend.util import tracing+++def _tool(body) -> BaseTool:+ class _Traced(BaseTool):+ @property+ def name(self) -> str:+ return "read_workspace_file"++ @property+ def description(self) -> str:+ return "traced"++ @property+ def parameters(self) -> dict:+ return {"type": "object", "properties": {}}++ async def _execute(self, user_id, session, **kwargs):+ return await body()++ return _Traced()+++async def test_a_tool_call_is_a_child_transaction_of_its_turn(sentry_transactions):+ async def body():+ with sentry_sdk.start_span(op="http.client", name="POST /get_graph"):+ return ErrorResponse(message="fine", session_id="s1")++ with tracing.start_transaction("copilot.turn", "copilot.turn") as turn:+ await _tool(body).execute("u1", MagicMock(session_id="s1"), "call-1")++ call, outer = sentry_transactions+ trace = call["contexts"]["trace"]+ assert call["transaction"] == "read_workspace_file"+ assert trace["op"] == "copilot.tool"+ assert trace["trace_id"] == turn.trace_id+ assert trace["parent_span_id"] == turn.span_id+ assert trace["data"]["copilot.tool_call_id"] == "call-1"+ assert call["tags"]["copilot.tool.success"] == "true"+ assert [span["op"] for span in call["spans"]] == ["http.client"]+ assert outer["spans"] == []+++async def test_a_tool_that_raises_fails_its_transaction(sentry_transactions):+ async def body():+ raise RuntimeError("tool crashed")++ result = await _tool(body).execute("u1", MagicMock(session_id="s1"), "call-2")++ assert result.success is False+ [call] = sentry_transactions+ assert call["contexts"]["trace"]["status"] == "internal_error"+ assert call["tags"]["copilot.tool.success"] == "false"diff --git a/autogpt_platform/backend/backend/copilot/turn_tracing.py b/autogpt_platform/backend/backend/copilot/turn_tracing.pynew file mode 100644index 0000000000..59403413f8--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/turn_tracing.py@@ -0,0 +1,143 @@+"""Sentry transactions for copilot turns and their tool calls.++A turn runs in the copilot executor as one ``copilot.turn`` transaction that+continues the trace of whatever dispatched it: the chat request, a parent+turn's tool, a graph node. Each tool call the turn makes is a ``copilot.tool``+transaction of its own in that trace, so a long autopilot turn stays under the+SDK's span limit however many tools it runs.+"""++import time+from collections.abc import Iterator+from contextlib import contextmanager+from typing import TYPE_CHECKING++from sentry_sdk.tracing import Span++from backend.copilot.response_model import (+ StreamBaseResponse,+ StreamReasoningDelta,+ StreamTextDelta,+ StreamToolInputAvailable,+ StreamToolInputStart,+)+from backend.util import tracing++if TYPE_CHECKING:+ from backend.copilot.executor.utils import CoPilotExecutionEntry++TURN_OP = "copilot.turn"+TOOL_OP = "copilot.tool"++MODEL_OUTPUT_CHUNKS = (StreamTextDelta, StreamReasoningDelta, StreamToolInputStart)+"""Chunks that carry model output. The first one is the turn's first token."""+++class TurnTrace:+ """One turn's timings and outcome, recorded on its transaction."""++ def __init__(self, span: Span) -> None:+ self.span = span+ self._started_at = time.monotonic()+ self._first_token_seen = False+ self._chunks = 0+ self._tool_calls = 0++ def engine_started(self, engine: str) -> None:+ """Setup is done and the turn is handed to ``engine``."""+ self.span.set_tag("copilot.engine", engine)+ tracing.set_timing("copilot.turn.setup_ms", self._elapsed_ms(), span=self.span)++ def observe(self, chunk: StreamBaseResponse) -> None:+ """Account for one chunk the turn streamed."""+ self._chunks += 1+ if isinstance(chunk, StreamToolInputAvailable):+ self._tool_calls += 1+ if not self._first_token_seen and isinstance(chunk, MODEL_OUTPUT_CHUNKS):+ self._first_token_seen = True+ tracing.set_timing(+ "copilot.turn.ttft_ms", self._elapsed_ms(), span=self.span+ )++ def set_outcome(self, error_message: str | None, cancelled_message: str) -> None:+ """Tag how the turn ended, given the error it reports to the user."""+ if not error_message:+ outcome, status = "completed", "ok"+ elif error_message == cancelled_message:+ outcome, status = "cancelled", "cancelled"+ else:+ outcome, status = "failed", "internal_error"+ self.span.set_tag("copilot.turn.outcome", outcome)+ tracing.set_status(status, span=self.span)++ def close(self) -> None:+ tracing.set_timing(+ "copilot.turn.duration_ms", self._elapsed_ms(), span=self.span+ )+ tracing.set_data("copilot.turn.chunk_count", self._chunks, span=self.span)+ tracing.set_data(+ "copilot.turn.tool_call_count", self._tool_calls, span=self.span+ )++ def _elapsed_ms(self) -> float:+ return (time.monotonic() - self._started_at) * 1000+++@contextmanager+def turn_transaction(entry: "CoPilotExecutionEntry") -> Iterator[TurnTrace]:+ """Run one turn as a ``copilot.turn`` transaction, continuing the trace+ the entry was dispatched with."""+ with tracing.start_transaction(+ TURN_OP,+ TURN_OP,+ trace_headers=entry.trace_headers,+ tags={+ "copilot.llm_auth_provider": entry.llm_auth_provider,+ "copilot.model_tier": entry.model or "default",+ "copilot.turn.origin": _origin(entry),+ },+ data={+ key: value+ for key, value in {+ "copilot.session_id": entry.session_id,+ "copilot.turn_id": entry.turn_id,+ "copilot.user_id": entry.user_id,+ }.items()+ if value+ },+ ) as span:+ if entry.enqueued_at:+ tracing.set_timing(+ "copilot.turn.queue_wait_ms",+ (time.time() - entry.enqueued_at) * 1000,+ span=span,+ )+ turn = TurnTrace(span)+ try:+ yield turn+ finally:+ turn.close()+++def _origin(entry: "CoPilotExecutionEntry") -> str:+ if entry.scheduled is not None or entry.unattended:+ return "scheduled"+ if entry.envelope is not None and entry.envelope.depth > 0:+ return "spawned"+ if not entry.is_user_message:+ return "system"+ return "chat"+++@contextmanager+def tool_transaction(+ tool_name: str, tool_call_id: str, session_id: str+) -> Iterator[Span]:+ """Run one tool call as a ``copilot.tool`` transaction named after the+ tool, as a child of the turn that made it."""+ with tracing.start_transaction(+ TOOL_OP,+ tool_name,+ data={"copilot.tool_call_id": tool_call_id, "copilot.session_id": session_id},+ ) as span:+ yield spandiff --git a/autogpt_platform/backend/backend/data/execution.py b/autogpt_platform/backend/backend/data/execution.pyindex 5ac703794c..755d046e9a 100644--- a/autogpt_platform/backend/backend/data/execution.py+++ b/autogpt_platform/backend/backend/data/execution.py@@ -1549,6 +1549,15 @@ class NodeExecutionEntry(BaseModel):# reconcile, leaving the delta computed against a different number).# Excluded from dumps — in-memory only, scoped to a single live execution.pre_flight_charge: Optional[int] = Field(default=None, exclude=True)+ # When the node was queued and by what: ``graph_start`` for a node queued+ # before this executor picked the run up (an entry node, or one left+ # queued by an interrupted run), ``node_output`` for one queued by an+ # upstream node's output during the run. In-memory only, for the node's+ # queue wait on its Sentry transaction.+ queued_at: Optional[datetime] = Field(default=None, exclude=True)+ queued_by: Optional[Literal["graph_start", "node_output"]] = Field(+ default=None, exclude=True+ )class ExecutionQueue(Generic[T]):diff --git a/autogpt_platform/backend/backend/executor/cluster_lock.py b/autogpt_platform/backend/backend/executor/cluster_lock.pyindex d38e0d0118..da4896395d 100644--- a/autogpt_platform/backend/backend/executor/cluster_lock.py+++ b/autogpt_platform/backend/backend/executor/cluster_lock.py@@ -10,6 +10,8 @@ from redis.exceptions import ClusterDownErrorfrom redis.exceptions import ConnectionError as RedisConnectionErrorfrom redis.exceptions import TimeoutError as RedisTimeoutError+from backend.util import tracing+if TYPE_CHECKING:from backend.data.redis_client import AsyncRedisClient, RedisClient@@ -104,6 +106,15 @@ class ClusterLock:return Nonedef refresh(self) -> bool:+ """Refresh lock TTL if we still own it; see ``_refresh``.++ A keepalive repeated for as long as the lock is held, so its Redis+ calls are kept out of the holder's Sentry transaction.+ """+ with tracing.suppress_child_spans():+ return self._refresh()++ def _refresh(self) -> bool:"""Refresh lock TTL if we still own it.Rate limited to at most once every timeout/10 seconds (minimum 1 second).@@ -244,6 +255,15 @@ class AsyncClusterLock:return Noneasync def refresh(self) -> bool:+ """Refresh lock TTL if we still own it; see ``_refresh``.++ A keepalive repeated for as long as the lock is held, so its Redis+ calls are kept out of the holder's Sentry transaction.+ """+ with tracing.suppress_child_spans():+ return await self._refresh()++ async def _refresh(self) -> bool:"""Refresh lock TTL if we still own it.Rate limited to at most once every timeout/10 seconds (minimum 1 second).diff --git a/autogpt_platform/backend/backend/executor/manager.py b/autogpt_platform/backend/backend/executor/manager.pyindex 36a07f9f50..eb555a79ef 100644--- a/autogpt_platform/backend/backend/executor/manager.py+++ b/autogpt_platform/backend/backend/executor/manager.py@@ -17,6 +17,7 @@ from redis.asyncio.lock import Lock as AsyncRedisLockfrom sentry_sdk.api import capture_exception as _sentry_capture_exceptionfrom sentry_sdk.api import flush as _sentry_flushfrom sentry_sdk.api import get_current_scope as _sentry_get_current_scope+from sentry_sdk.tracing import Spanfrom backend.blocks import get_block, get_blocksfrom backend.blocks._base import BlockSchema@@ -56,7 +57,7 @@ from backend.integrations.credential_lease import CredentialLeasefrom backend.integrations.credentials_store import provider_matchesfrom backend.integrations.creds_manager import IntegrationCredentialsManagerfrom backend.monitoring.instrumentation import record_graph_run_completion-from backend.util import json, product_analytics+from backend.util import json, product_analytics, tracingfrom backend.util.clients import (get_async_execution_event_bus,get_database_manager_async_client,@@ -96,6 +97,12 @@ from .activity_status_generator import (from .auto_credentials import acquire_auto_credentialsfrom .automod.manager import automod_managerfrom .cluster_lock import ClusterLock+from .node_tracing import (+ node_transaction,+ output_trace_headers,+ output_transaction,+ record_node_result,+)from .simulator import get_dry_run_credentials, prepare_dry_run, simulate_blockfrom .utils import (GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS,@@ -113,6 +120,7 @@ from .utils import ()if TYPE_CHECKING:+ from backend.blocks._base import AnyBlockSchemafrom backend.data.db_manager import (DatabaseManagerAsyncClient,DatabaseManagerClient,@@ -580,6 +588,8 @@ async def _enqueue_next_nodes(block_id=block_id,inputs=data,execution_context=execution_context,+ queued_at=datetime.now(timezone.utc),+ queued_by="node_output",)async def register_next_executions(node_link: Link) -> list[NodeExecutionEntry]:@@ -745,6 +755,30 @@ class ExecutionProcessor:nodes_input_masks: Optional[NodesInputMasks],graph_stats_pair: tuple[GraphExecutionStats, threading.Lock],nodes_to_skip: Optional[set[str]] = None,+ ) -> NodeExecutionStats:+ """Run one node and account for it, as one Sentry ``executor.node``+ transaction named after its block."""+ block = get_block(node_exec.block_id)+ with node_transaction(node_exec, block) as span:+ return await self._run_node(+ node_exec,+ block,+ node_exec_progress,+ nodes_input_masks,+ graph_stats_pair,+ nodes_to_skip,+ span,+ )++ async def _run_node(+ self,+ node_exec: NodeExecutionEntry,+ block: "AnyBlockSchema | None",+ node_exec_progress: NodeExecutionProgress,+ nodes_input_masks: Optional[NodesInputMasks],+ graph_stats_pair: tuple[GraphExecutionStats, threading.Lock],+ nodes_to_skip: Optional[set[str]],+ span: Span,) -> NodeExecutionStats:log_metadata = LogMetadata(logger=_logger,@@ -753,7 +787,7 @@ class ExecutionProcessor:graph_id=node_exec.graph_id,node_eid=node_exec.node_exec_id,node_id=node_exec.node_id,- block_name=b.name if (b := get_block(node_exec.block_id)) else "-",+ block_name=block.name if block else "-",)db_client = get_db_async_client()node = await db_client.get_node(node_exec.node_id)@@ -774,6 +808,7 @@ class ExecutionProcessor:execution_stats.walltime = timing_info.wall_timeexecution_stats.cputime = timing_info.cpu_time+ record_node_result(span, status, execution_stats)# Log platform cost + reconcile dynamic billing BEFORE graph/node stats# are aggregated and persisted — otherwise the reconciled delta never@@ -867,8 +902,11 @@ class ExecutionProcessor:nodes_to_skip: Optional[set[str]] = None,) -> ExecutionStatus:status = ExecutionStatus.RUNNING+ outputs = 0async def persist_output(output_name: str, output_data: Any) -> None:+ nonlocal outputs+ outputs += 1await db_client.upsert_execution_output(node_exec_id=node_exec.node_exec_id,output_name=output_name,@@ -884,6 +922,7 @@ class ExecutionProcessor:node=node,node_exec_id=node_exec.node_exec_id,data=(output_name, output_data),+ trace_headers=output_trace_headers(outputs),))@@ -958,6 +997,7 @@ class ExecutionProcessor:await persist_output("error", str(stats.error) or type(stats.error).__name__)+ tracing.set_data("executor.node.output_count", outputs)return status@func_retry@@ -1196,6 +1236,8 @@ class ExecutionProcessor:node_entry = node_exec.to_node_execution_entry(graph_exec.execution_context)+ node_entry.queued_at = node_exec.queue_time+ node_entry.queued_by = "graph_start"execution_queue.add(node_entry)# ------------------------------------------------------------@@ -1508,7 +1550,8 @@ class ExecutionProcessor:nodes_input_masks: Optional[NodesInputMasks],execution_queue: ExecutionQueue[NodeExecutionEntry],) -> None:- """Process a node's output, update its status, and enqueue next nodes.+ """Process a node's output, update its status, and enqueue next nodes,+ as one Sentry ``executor.node.output`` transaction.Args:output: The execution output entry to process@@ -1518,23 +1561,28 @@ class ExecutionProcessor:nodes_input_masks: Optional map of node input overridesexecution_queue: Queue to add next executions to"""- db_client = get_db_async_client()+ with output_transaction(output, graph_exec) as span:+ db_client = get_db_async_client()- log_metadata.debug(f"Enqueue nodes for {node_id}: {output}")+ log_metadata.debug(f"Enqueue nodes for {node_id}: {output}")- for next_execution in await _enqueue_next_nodes(- db_client=db_client,- node=output.node,- output=output.data,- user_id=graph_exec.user_id,- graph_exec_id=graph_exec.graph_exec_id,- graph_id=graph_exec.graph_id,- graph_version=graph_exec.graph_version,- log_metadata=log_metadata,- nodes_input_masks=nodes_input_masks,- execution_context=graph_exec.execution_context,- ):- execution_queue.add(next_execution)+ next_executions = await _enqueue_next_nodes(+ db_client=db_client,+ node=output.node,+ output=output.data,+ user_id=graph_exec.user_id,+ graph_exec_id=graph_exec.graph_exec_id,+ graph_id=graph_exec.graph_id,+ graph_version=graph_exec.graph_version,+ log_metadata=log_metadata,+ nodes_input_masks=nodes_input_masks,+ execution_context=graph_exec.execution_context,+ )+ tracing.set_data(+ "executor.node.output.enqueued_count", len(next_executions), span=span+ )+ for next_execution in next_executions:+ execution_queue.add(next_execution)class ExecutionManager(AppProcess):diff --git a/autogpt_platform/backend/backend/executor/node_tracing.py b/autogpt_platform/backend/backend/executor/node_tracing.pynew file mode 100644index 0000000000..625db2f1ef--- /dev/null+++ b/autogpt_platform/backend/backend/executor/node_tracing.py@@ -0,0 +1,136 @@+"""Sentry transactions for graph node runs.++Every node run is one ``executor.node`` transaction named after its block, and+the routing of each of its first outputs (persisting downstream inputs,+queueing the nodes they complete) is one ``executor.node.output`` transaction.+All of a graph execution's transactions, and those of the sub-graphs it+starts, share one trace whose id is the root execution's id without dashes,+so a run's trace can be opened from its id. A node run inside another node's+(an orchestrator's tool) is that run's child.+"""++import time+from collections.abc import Iterator+from contextlib import contextmanager+from datetime import datetime, timezone+from typing import TYPE_CHECKING++from sentry_sdk.tracing import NoOpSpan, Span++from backend.data.execution import (+ ExecutionContext,+ ExecutionStatus,+ GraphExecutionEntry,+ NodeExecutionEntry,+)+from backend.util import tracing++if TYPE_CHECKING:+ from backend.blocks._base import AnyBlockSchema+ from backend.data.model import NodeExecutionStats+ from backend.executor.utils import ExecutionOutputEntry++NODE_OP = "executor.node"+OUTPUT_OP = "executor.node.output"++TRACED_OUTPUTS_PER_NODE_RUN = 20+"""How many of a node run's outputs get their routing traced. A node can yield+thousands (an iterator over a list), and a transaction each would make the+run's tracing cost grow with its output instead of staying bounded."""+++def graph_trace_id(graph_exec_id: str, execution_context: ExecutionContext) -> str:+ """The trace shared by a graph execution and the sub-graphs it starts."""+ return tracing.trace_id_for(execution_context.root_execution_id or graph_exec_id)+++@contextmanager+def node_transaction(+ node_exec: NodeExecutionEntry, block: "AnyBlockSchema | None"+) -> Iterator[Span]:+ """Run one node execution as an ``executor.node`` transaction."""+ context = node_exec.execution_context+ with tracing.start_transaction(+ NODE_OP,+ block.name if block else "UnknownBlock",+ trace_id=graph_trace_id(node_exec.graph_exec_id, context),+ tags={+ "executor.block_type": block.block_type.value if block else "unknown",+ "executor.dry_run": "true" if context.dry_run else "false",+ },+ data={+ "executor.graph_exec_id": node_exec.graph_exec_id,+ "executor.node_exec_id": node_exec.node_exec_id,+ "executor.graph_id": node_exec.graph_id,+ "executor.graph_version": node_exec.graph_version,+ "executor.node_id": node_exec.node_id,+ "executor.block_id": node_exec.block_id,+ "executor.user_id": node_exec.user_id,+ },+ ) as span:+ if node_exec.queued_at is not None:+ queue_wait = datetime.now(timezone.utc) - node_exec.queued_at+ tracing.set_timing(+ "executor.node.queue_wait_ms",+ queue_wait.total_seconds() * 1000,+ span=span,+ )+ if node_exec.queued_by is not None:+ span.set_tag("executor.node.queued_by", node_exec.queued_by)+ yield span+++def record_node_result(+ span: Span, status: ExecutionStatus, stats: "NodeExecutionStats"+) -> None:+ """Record how a node run ended and how long its block ran."""+ tracing.set_timing("executor.node.run_ms", stats.walltime * 1000, span=span)+ span.set_tag("executor.node.status", status.value.lower())+ if status == ExecutionStatus.FAILED:+ # A ValueError is the executor's mark of an expected, user-caused+ # failure (bad input, missing credentials) rather than a platform one.+ failed_by_user = isinstance(stats.error, ValueError)+ tracing.set_status(+ "invalid_argument" if failed_by_user else "internal_error", span=span+ )+ elif status == ExecutionStatus.TERMINATED:+ tracing.set_status("cancelled", span=span)+++def output_trace_headers(output_number: int) -> dict[str, str]:+ """The trace the routing of a node run's ``output_number``-th output+ continues: the node run's, for its first ``TRACED_OUTPUTS_PER_NODE_RUN``+ outputs, and none after, so their routing runs untraced."""+ if output_number > TRACED_OUTPUTS_PER_NODE_RUN:+ return {}+ return tracing.trace_propagation_headers()+++@contextmanager+def output_transaction(+ output: "ExecutionOutputEntry", graph_exec: GraphExecutionEntry+) -> Iterator[Span]:+ """Run the routing of one node output as an ``executor.node.output``+ transaction, a child of the node run that produced it, when that run+ handed the output its trace (see ``output_trace_headers``)."""+ if not output.trace_headers:+ yield NoOpSpan()+ return+ block = output.node.block+ with tracing.start_transaction(+ OUTPUT_OP,+ OUTPUT_OP,+ trace_headers=output.trace_headers,+ tags={"executor.block": block.name if block else "UnknownBlock"},+ data={+ "executor.graph_exec_id": graph_exec.graph_exec_id,+ "executor.node_exec_id": output.node_exec_id,+ "executor.output_name": output.data[0],+ },+ ) as span:+ tracing.set_timing(+ "executor.node.output.wait_ms",+ (time.monotonic() - output.produced_at) * 1000,+ span=span,+ )+ yield spandiff --git a/autogpt_platform/backend/backend/executor/node_tracing_test.py b/autogpt_platform/backend/backend/executor/node_tracing_test.pynew file mode 100644index 0000000000..a476d4cb55--- /dev/null+++ b/autogpt_platform/backend/backend/executor/node_tracing_test.py@@ -0,0 +1,367 @@+"""Tests for the Sentry transactions of graph node runs and output routing.++They run against a live SDK client (the ``sentry_transactions`` fixture) and+assert on the transactions it would have sent.+"""++import logging+import threading+import uuid+from datetime import datetime, timedelta, timezone+from types import SimpleNamespace+from unittest.mock import AsyncMock, MagicMock, patch++import pytest+import sentry_sdk++from backend.blocks._base import BlockType+from backend.data.execution import (+ ExecutionContext,+ ExecutionQueue,+ ExecutionStatus,+ GraphExecutionEntry,+ NodeExecutionEntry,+)+from backend.data.graph import Node+from backend.data.model import GraphExecutionStats, NodeExecutionStats+from backend.executor import node_tracing+from backend.executor.manager import ExecutionProcessor+from backend.executor.utils import (+ ExecutionOutputEntry,+ LogMetadata,+ NodeExecutionProgress,+)+from backend.util import tracing+from backend.util.decorator import TimingInfo++ROOT_EXEC_ID = str(uuid.uuid4())+SUB_EXEC_ID = str(uuid.uuid4())++_AI_BLOCK = SimpleNamespace(name="AITextGeneratorBlock", block_type=BlockType.AI)+++def _node_exec(graph_exec_id: str = ROOT_EXEC_ID, **context) -> NodeExecutionEntry:+ return NodeExecutionEntry(+ user_id="user-1",+ graph_exec_id=graph_exec_id,+ graph_id="graph-1",+ graph_version=3,+ node_exec_id="node-exec-1",+ node_id="node-1",+ block_id="block-1",+ inputs={},+ execution_context=ExecutionContext(+ root_execution_id=context.pop("root_execution_id", graph_exec_id),+ **context,+ ),+ )+++def _trace(transaction: dict) -> dict:+ return transaction["contexts"]["trace"]+++# ---------- node_transaction ----------+++def test_a_node_run_is_a_transaction_named_after_its_block(sentry_transactions):+ with node_tracing.node_transaction(_node_exec(), _AI_BLOCK):+ pass++ [run] = sentry_transactions+ assert run["transaction"] == "AITextGeneratorBlock"+ assert _trace(run)["op"] == "executor.node"+ assert run["tags"] == {"executor.block_type": "AI", "executor.dry_run": "false"}+ data = _trace(run)["data"]+ assert data["executor.graph_exec_id"] == ROOT_EXEC_ID+ assert data["executor.node_exec_id"] == "node-exec-1"+ assert data["executor.block_id"] == "block-1"+ assert data["executor.user_id"] == "user-1"+++def test_every_node_run_of_a_run_shares_the_runs_trace(sentry_transactions):+ with node_tracing.node_transaction(_node_exec(), _AI_BLOCK):+ pass+ with node_tracing.node_transaction(+ _node_exec(SUB_EXEC_ID, root_execution_id=ROOT_EXEC_ID), _AI_BLOCK+ ):+ pass++ root_trace = uuid.UUID(ROOT_EXEC_ID).hex+ assert [_trace(t)["trace_id"] for t in sentry_transactions] == [root_trace] * 2+++def test_a_node_run_inside_another_is_its_child(sentry_transactions):+ """An orchestrator runs its tool nodes from inside its own node run."""+ with node_tracing.node_transaction(_node_exec(), _AI_BLOCK) as orchestrator:+ with node_tracing.node_transaction(_node_exec(), _AI_BLOCK):+ pass++ tool, _ = sentry_transactions+ assert _trace(tool)["parent_span_id"] == orchestrator.span_id+++def test_the_queue_wait_runs_from_when_the_node_was_queued(sentry_transactions):+ node_exec = _node_exec()+ node_exec.queued_at = datetime.now(timezone.utc) - timedelta(seconds=3)+ node_exec.queued_by = "node_output"++ with node_tracing.node_transaction(node_exec, _AI_BLOCK):+ pass++ [run] = sentry_transactions+ assert 3000 <= _trace(run)["data"]["executor.node.queue_wait_ms"] < 60_000+ assert run["tags"]["executor.node.queued_by"] == "node_output"+++def test_an_unknown_block_still_gets_a_low_cardinality_name(sentry_transactions):+ with node_tracing.node_transaction(_node_exec(), None):+ pass++ assert sentry_transactions[0]["transaction"] == "UnknownBlock"+++# ---------- record_node_result ----------+++ "status, error, span_status",+ [+ (ExecutionStatus.COMPLETED, None, "ok"),+ (ExecutionStatus.FAILED, ValueError("missing credentials"), "invalid_argument"),+ (ExecutionStatus.FAILED, RuntimeError("provider down"), "internal_error"),+ (ExecutionStatus.TERMINATED, None, "cancelled"),+ ],+)+def test_how_a_node_run_ended_is_recorded(+ sentry_transactions, status, error, span_status+):+ stats = NodeExecutionStats(walltime=1.25)+ stats.error = error++ with node_tracing.node_transaction(_node_exec(), _AI_BLOCK) as span:+ node_tracing.record_node_result(span, status, stats)++ [run] = sentry_transactions+ assert _trace(run)["status"] == span_status+ assert _trace(run)["data"]["executor.node.run_ms"] == 1250+ assert run["tags"]["executor.node.status"] == status.value.lower()+++# ---------- ExecutionProcessor ----------+++def _db_client() -> MagicMock:+ client = MagicMock()+ client.get_node = AsyncMock(return_value=MagicMock(block=_AI_BLOCK))+ return client+++async def test_on_node_execution_runs_the_node_as_one_transaction(sentry_transactions):+ processor = ExecutionProcessor()++ async def run_block(**kwargs):+ with sentry_sdk.start_span(op="http.client", name="POST /upsert_output"):+ return TimingInfo(cpu_time=0.1, wall_time=2.0), ExecutionStatus.COMPLETED++ processor._on_node_execution = run_block # type: ignore[method-assign]+ node_exec = _node_exec()+ node_exec.queued_at = datetime.now(timezone.utc)+ node_exec.queued_by = "graph_start"++ with (+ patch("backend.executor.manager.get_block", return_value=_AI_BLOCK),+ patch(+ "backend.executor.manager.get_db_async_client", return_value=_db_client()+ ),+ patch("backend.executor.manager.log_system_credential_cost", new=AsyncMock()),+ patch(+ "backend.executor.manager.activity_events.log_node_integration_activity",+ new=AsyncMock(),+ ),+ patch(+ "backend.executor.manager.billing.charge_reconciled_usage",+ new=AsyncMock(return_value=(0, None)),+ ),+ patch(+ "backend.executor.manager.async_update_node_execution_status",+ new=AsyncMock(),+ ),+ patch(+ "backend.executor.manager.async_update_graph_execution_state",+ new=AsyncMock(),+ ),+ ):+ stats = await processor.on_node_execution(+ node_exec=node_exec,+ node_exec_progress=NodeExecutionProgress(),+ nodes_input_masks=None,+ graph_stats_pair=(GraphExecutionStats(), threading.Lock()),+ )++ assert stats is not None and stats.walltime == 2.0+ [run] = sentry_transactions+ assert run["transaction"] == "AITextGeneratorBlock"+ assert _trace(run)["trace_id"] == uuid.UUID(ROOT_EXEC_ID).hex+ assert _trace(run)["status"] == "ok"+ assert _trace(run)["data"]["executor.node.run_ms"] == 2000+ assert "executor.node.queue_wait_ms" in _trace(run)["data"]+ assert run["tags"]["executor.node.queued_by"] == "graph_start"+ assert [span["op"] for span in run["spans"]] == ["http.client"]+++def _graph_exec() -> GraphExecutionEntry:+ return GraphExecutionEntry(+ user_id="user-1",+ graph_exec_id=ROOT_EXEC_ID,+ graph_id="graph-1",+ graph_version=3,+ execution_context=ExecutionContext(root_execution_id=ROOT_EXEC_ID),+ )+++def _log_metadata() -> LogMetadata:+ return LogMetadata(+ logger=logging.getLogger("test"),+ user_id="user-1",+ graph_eid=ROOT_EXEC_ID,+ graph_id="graph-1",+ node_eid="node-exec-1",+ node_id="node-1",+ block_name="AITextGeneratorBlock",+ )+++async def test_routing_an_output_is_a_child_of_the_node_run(sentry_transactions):+ processor = ExecutionProcessor()+ with node_tracing.node_transaction(_node_exec(), _AI_BLOCK) as node_run:+ output = ExecutionOutputEntry(+ node=Node(id="node-1", block_id="block-1"),+ node_exec_id="node-exec-1",+ data=("response", "hello"),+ trace_headers=tracing.trace_propagation_headers(),+ )+ queue: ExecutionQueue[NodeExecutionEntry] = ExecutionQueue()++ with (+ patch("backend.data.graph.get_block", return_value=_AI_BLOCK),+ patch("backend.executor.manager.get_db_async_client"),+ patch(+ "backend.executor.manager._enqueue_next_nodes",+ new=AsyncMock(return_value=[_node_exec(), _node_exec()]),+ ),+ ):+ await processor._process_node_output(+ output=output,+ node_id="node-1",+ graph_exec=_graph_exec(),+ log_metadata=_log_metadata(),+ nodes_input_masks=None,+ execution_queue=queue,+ )++ assert queue.get_or_none() is not None and queue.get_or_none() is not None+ routing = next(+ t for t in sentry_transactions if _trace(t)["op"] == "executor.node.output"+ )+ assert routing["transaction"] == "executor.node.output"+ assert _trace(routing)["trace_id"] == node_run.trace_id+ assert _trace(routing)["parent_span_id"] == node_run.span_id+ assert routing["tags"] == {"executor.block": "AITextGeneratorBlock"}+ data = _trace(routing)["data"]+ assert data["executor.node.output.enqueued_count"] == 2+ assert data["executor.output_name"] == "response"+ assert data["executor.node.output.wait_ms"] >= 0+++def test_a_node_run_hands_its_trace_to_its_first_outputs_only(sentry_transactions):+ limit = node_tracing.TRACED_OUTPUTS_PER_NODE_RUN++ with node_tracing.node_transaction(_node_exec(), _AI_BLOCK):+ within = [node_tracing.output_trace_headers(n) for n in (1, limit)]+ past = node_tracing.output_trace_headers(limit + 1)++ assert all(headers.get("sentry-trace") for headers in within)+ assert past == {}+++async def test_a_high_fanout_node_traces_a_bounded_number_of_routings(+ sentry_transactions,+):+ """A node yielding many outputs (an iterator over a list) hands its trace to+ the first few only, and its transaction still counts them all."""+ processor = ExecutionProcessor()+ progress = NodeExecutionProgress()+ produced = node_tracing.TRACED_OUTPUTS_PER_NODE_RUN + 5++ async def yield_items(**kwargs):+ for item in range(produced):+ yield "item", item++ db_client = MagicMock(+ upsert_execution_output=AsyncMock(),+ get_node_execution=AsyncMock(return_value=None),+ )+ node = Node(id="node-1", block_id="block-1")+ untimed_block = SimpleNamespace(execution_timeout_seconds=None)++ with (+ patch("backend.data.graph.get_block", return_value=untimed_block),+ patch("backend.executor.manager.execute_node", new=yield_items),+ patch(+ "backend.executor.manager.async_update_node_execution_status",+ new=AsyncMock(),+ ),+ ):+ with node_tracing.node_transaction(_node_exec(), _AI_BLOCK):+ _, status = await processor._on_node_execution(+ node=node,+ node_exec=_node_exec(),+ node_exec_progress=progress,+ stats=NodeExecutionStats(),+ db_client=db_client,+ log_metadata=_log_metadata(),+ )++ assert status == ExecutionStatus.COMPLETED+ outputs = progress.output["node-exec-1"]+ assert len(outputs) == produced+ assert [bool(o.trace_headers) for o in outputs] == [True] * (+ node_tracing.TRACED_OUTPUTS_PER_NODE_RUN+ ) + [False] * 5+ [run] = sentry_transactions+ assert _trace(run)["data"]["executor.node.output_count"] == produced+++async def test_an_output_the_node_run_did_not_trace_is_routed_untraced(+ sentry_transactions,+):+ queue: ExecutionQueue[NodeExecutionEntry] = ExecutionQueue()+ untraced = ExecutionOutputEntry(+ node=Node(id="node-1", block_id="block-1"),+ node_exec_id="node-exec-1",+ data=("item", 99),+ )++ with (+ patch("backend.executor.manager.get_db_async_client"),+ patch(+ "backend.executor.manager._enqueue_next_nodes",+ new=AsyncMock(return_value=[_node_exec()]),+ ),+ ):+ await ExecutionProcessor()._process_node_output(+ output=untraced,+ node_id="node-1",+ graph_exec=_graph_exec(),+ log_metadata=_log_metadata(),+ nodes_input_masks=None,+ execution_queue=queue,+ )++ assert queue.get_or_none() is not None+ assert sentry_transactions == []diff --git a/autogpt_platform/backend/backend/executor/utils.py b/autogpt_platform/backend/backend/executor/utils.pyindex 7d5a550a01..71dd039655 100644--- a/autogpt_platform/backend/backend/executor/utils.py+++ b/autogpt_platform/backend/backend/executor/utils.py@@ -8,7 +8,7 @@ from concurrent.futures import Futurefrom typing import Literal, Mapping, Optional, cast, get_argsfrom prisma.enums import ReviewStatus-from pydantic import BaseModel, JsonValue, ValidationError+from pydantic import BaseModel, Field, JsonValue, ValidationErrorfrom backend.api.features.experts import scheduling as experts_schedulingfrom backend.blocks import get_block@@ -1830,6 +1830,10 @@ class ExecutionOutputEntry(BaseModel):node: Nodenode_exec_id: strdata: BlockOutputEntry+ # The producing node's Sentry trace, which the output's routing continues,+ # and when the output was produced (monotonic), for the routing's wait.+ trace_headers: dict[str, str] = Field(default_factory=dict, exclude=True)+ produced_at: float = Field(default_factory=time.monotonic, exclude=True)class NodeExecutionProgress:diff --git a/autogpt_platform/backend/backend/util/cache.py b/autogpt_platform/backend/backend/util/cache.pyindex 85126c2a83..648c3c534d 100644--- a/autogpt_platform/backend/backend/util/cache.py+++ b/autogpt_platform/backend/backend/util/cache.py@@ -21,8 +21,10 @@ from dataclasses import dataclassfrom functools import cache, wrapsfrom typing import Any, Callable, ParamSpec, Protocol, TypeVar, cast, runtime_checkable+import sentry_sdkfrom redis.cluster import ClusterNode, RedisCluster+from backend.util import tracingfrom backend.util.retry import conn_retryfrom backend.util.settings import Settings@@ -404,11 +406,17 @@ def cached(if patternelse f"cache:{func_name}:*")- keys = list(- _get_redis().scan_iter(- match_pattern, target_nodes=RedisCluster.PRIMARIES+ # One span for the whole sweep: a span per SCAN page fills the+ # caller's transaction past the SDK's span limit.+ with (+ sentry_sdk.start_span(op="db.redis", name="SCAN cache keys"),+ tracing.suppress_child_spans(),+ ):+ keys = list(+ _get_redis().scan_iter(+ match_pattern, target_nodes=RedisCluster.PRIMARIES+ ))- )if keys:pipeline = _get_redis().pipeline()diff --git a/autogpt_platform/backend/backend/util/cache_tracing_test.py b/autogpt_platform/backend/backend/util/cache_tracing_test.pynew file mode 100644index 0000000000..d9cb6f5334--- /dev/null+++ b/autogpt_platform/backend/backend/util/cache_tracing_test.py@@ -0,0 +1,60 @@+"""How a shared cache's operations show up on the caller's Sentry transaction."""++import sentry_sdk++from backend.util import cache as cache_module+from backend.util import tracing+from backend.util.cache import cached+++class _SpanPerPageRedis:+ """A shared-cache Redis with Sentry's integration on: each SCAN page and+ pipeline execution is a ``db.redis`` span."""++ def __init__(self, keys: list[str], page_size: int):+ self._keys = keys+ self._page_size = page_size+ self.deleted: list[str] = []++ def scan_iter(self, match, target_nodes=None):+ for start in range(0, len(self._keys), self._page_size):+ with sentry_sdk.start_span(op="db.redis", name=f"SCAN {start}"):+ page = self._keys[start : start + self._page_size]+ yield from page++ def pipeline(self):+ deleted = self.deleted++ class _Pipeline:+ def delete(self, key):+ deleted.append(key)++ def execute(self):+ with sentry_sdk.start_span(op="db.redis", name="pipeline"):+ pass++ return _Pipeline()+++def test_clearing_a_shared_cache_is_one_span_however_many_pages(+ sentry_transactions, monkeypatch+):+ """The sweep reads every primary page by page; a span per page took the+ caller's transaction past the SDK's span limit."""+ keys = [f"cache:swept:{i}" for i in range(50)]+ redis = _SpanPerPageRedis(keys, page_size=2)+ monkeypatch.setattr(cache_module, "_get_redis", lambda: redis)++ @cached(ttl_seconds=30, shared_cache=True)+ def swept(x: int) -> int:+ return x++ with tracing.start_transaction("http.server", "/api/auth/user/timezone"):+ swept.cache_clear()++ assert redis.deleted == keys+ [request] = sentry_transactions+ assert [span["description"] for span in request["spans"]] == [+ "SCAN cache keys",+ "pipeline",+ ]diff --git a/autogpt_platform/backend/backend/util/metrics.py b/autogpt_platform/backend/backend/util/metrics.pyindex c8f80645d1..c2638621fb 100644--- a/autogpt_platform/backend/backend/util/metrics.py+++ b/autogpt_platform/backend/backend/util/metrics.py@@ -20,6 +20,7 @@ except ImportError:from backend.util.exceptions import get_execution_failure_reasonfrom backend.util.security import SENSITIVE_FIELD_NAMESfrom backend.util.settings import BehaveAs, Settings+from backend.util.tracing import traces_samplerfrom backend.util.url_redaction import REDACTED, redact_urlsettings = Settings()@@ -308,6 +309,11 @@ def sentry_init():_sentry_init(dsn=sentry_dsn,traces_sample_rate=1.0,+ traces_sampler=traces_sampler,+ # Traces reach our own services only, through the RPC client's headers+ # (backend.util.service); outbound calls to third parties, such as a+ # block's request to a user-supplied URL, carry no Sentry headers.+ trace_propagation_targets=[],profiles_sample_rate=1.0,environment=f"app:{settings.config.app_env.value}-behave:{settings.config.behave_as.value}",before_send=_before_send,diff --git a/autogpt_platform/backend/backend/util/metrics_test.py b/autogpt_platform/backend/backend/util/metrics_test.pyindex 0f3e2ea1cc..327123188e 100644--- a/autogpt_platform/backend/backend/util/metrics_test.py+++ b/autogpt_platform/backend/backend/util/metrics_test.py@@ -22,7 +22,7 @@ from sentry_sdk.utils import event_from_exception, json_dumps# Imported at module scope on purpose: AppProcess calls sentry_init() in its# class body, so the guard has to hold at collection time, not just in a test.import backend.util.process-from backend.util import feature_flag, metrics+from backend.util import feature_flag, metrics, tracingfrom backend.util.exceptions import InsufficientBalanceErrorfrom backend.util.metrics import (_FALKORDB_DRIVER_LOGGER,@@ -401,6 +401,30 @@ def test_sentry_init_runs_outside_pytest(monkeypatch) -> None:assert calls[0]["dsn"] == _FAKE_DSN+def test_sentry_init_samples_through_the_shared_sampler(monkeypatch) -> None:+ """Probes, scrapes and WebSocket connections are dropped by the sampler;+ see tracing_test for its rules."""+ calls = _spy_on_sentry_init(monkeypatch)+ monkeypatch.delitem(sys.modules, "pytest")+ monkeypatch.delenv("PYTEST_CURRENT_TEST", raising=False)++ metrics.sentry_init()++ assert calls[0]["traces_sampler"] is tracing.traces_sampler+++def test_sentry_init_sends_no_trace_headers_to_third_parties(monkeypatch) -> None:+ """Only the RPC client attaches trace headers (service_test covers it), so a+ block calling a user-supplied URL never hands it Sentry's trace or key."""+ calls = _spy_on_sentry_init(monkeypatch)+ monkeypatch.delitem(sys.modules, "pytest")+ monkeypatch.delenv("PYTEST_CURRENT_TEST", raising=False)++ metrics.sentry_init()++ assert calls[0]["trace_propagation_targets"] == []++def test_sentry_init_skipped_in_subprocess_spawned_by_pytest(monkeypatch) -> None:"""A spawned service subprocess does not inherit sys.modules, but doesinherit PYTEST_CURRENT_TEST from the environment."""diff --git a/autogpt_platform/backend/backend/util/service.py b/autogpt_platform/backend/backend/util/service.pyindex 48b5fae896..3394b6c0ae 100644--- a/autogpt_platform/backend/backend/util/service.py+++ b/autogpt_platform/backend/backend/util/service.py@@ -37,6 +37,7 @@ from sentry_sdk.api import capture_exception as _sentry_capture_exceptionimport backend.util.exceptions as exceptionsfrom backend.data import redis_clientfrom backend.monitoring.instrumentation import instrument_fastapi+from backend.util import tracingfrom backend.util.json import to_dictfrom backend.util.metrics import sentry_initfrom backend.util.process import AppProcess@@ -602,6 +603,16 @@ def _usable_adapter(annotation: Any) -> TypeAdapter | None:return None+def _send_trace_headers(request: httpx.Request) -> None:+ """Carry the caller's trace to the service, whose request transaction then+ continues it. Runs inside the HTTP client span, which becomes the parent."""+ request.headers.update(tracing.trace_propagation_headers())+++async def _send_trace_headers_async(request: httpx.Request) -> None:+ _send_trace_headers(request)++@conn_retry("AppService client", "Creating service client", max_retry=api_comm_retry)def get_service_client(service_client_type: Type[ASC],@@ -653,6 +664,7 @@ def get_service_client(max_connections=500, # High limit for burst handlingkeepalive_expiry=30.0, # Keep connections alive longer),+ event_hooks={"request": [_send_trace_headers]},)def _create_async_client(self) -> httpx.AsyncClient:@@ -664,6 +676,7 @@ def get_service_client(max_connections=500, # High limit for burst handlingkeepalive_expiry=30.0, # Keep connections alive longer),+ event_hooks={"request": [_send_trace_headers_async]},)@propertydiff --git a/autogpt_platform/backend/backend/util/service_test.py b/autogpt_platform/backend/backend/util/service_test.pyindex 426893c944..ba6068139e 100644--- a/autogpt_platform/backend/backend/util/service_test.py+++ b/autogpt_platform/backend/backend/util/service_test.py@@ -17,6 +17,7 @@ from prisma.errors import DataError, UniqueViolationErrorfrom pydantic import BaseModel, Field, TypeAdapter, ValidationErrorfrom backend.data.model import User+from backend.util import tracingfrom backend.util.exceptions import (ExpertNotFoundError,ExpertPrivateTenancyNotFoundError,@@ -1153,3 +1154,59 @@ async def test_forward_ref_return_is_deserialized_over_the_async_rpc():assert reviews == [ReviewLike(id="0")]assert isinstance(reviews[0], ReviewLike)+++def _client_recording_requests(+ loop: asyncio.AbstractEventLoop | None = None,+) -> tuple[Any, list[httpx.Request]]:+ """A client built as in production, with only its network layer swapped+ for one that records each request and answers 8."""+ sent: list[httpx.Request] = []++ def answer(request: httpx.Request) -> httpx.Response:+ sent.append(request)+ return httpx.Response(200, json=8)++ client = get_service_client(ServiceTestClient)+ inner = cast(Any, client)+ sync_client = inner._create_sync_client()+ sync_client._transport = httpx.MockTransport(answer)+ inner._sync_clients[ServiceTest.__name__] = sync_client+ async_client = inner._create_async_client()+ async_client._transport = httpx.MockTransport(answer)+ inner._async_clients[loop] = async_client+ return client, sent+++def test_an_rpc_carries_the_callers_trace(sentry_transactions):+ client, sent = _client_recording_requests()++ with tracing.start_transaction("executor.node", "Block") as transaction:+ assert client.add(5, 3) == 8++ [request] = sent+ assert request.headers["sentry-trace"].startswith(transaction.trace_id)+ assert f"sentry-trace_id={transaction.trace_id}" in request.headers["baggage"]+++async def test_an_async_rpc_carries_the_callers_trace(sentry_transactions):+ client, sent = _client_recording_requests(asyncio.get_running_loop())++ with tracing.start_transaction("executor.node", "Block") as transaction:+ assert await client.add_async(5, 3) == 8++ [request] = sent+ assert request.headers["sentry-trace"].startswith(transaction.trace_id)+++def test_an_untraced_rpc_carries_no_trace(sentry_transactions):+ """Without a transaction the service starts a trace of its own rather than+ joining whatever trace the calling thread last saw."""+ client, sent = _client_recording_requests()++ assert client.add(5, 3) == 8++ [request] = sent+ assert "sentry-trace" not in request.headers+ assert "baggage" not in request.headersdiff --git a/autogpt_platform/backend/backend/util/tracing.py b/autogpt_platform/backend/backend/util/tracing.pynew file mode 100644index 0000000000..214bf9c0d8--- /dev/null+++ b/autogpt_platform/backend/backend/util/tracing.py@@ -0,0 +1,332 @@+"""One convention for Sentry transactions and the timings recorded on them.++HTTP requests get a transaction from the SDK's integrations. Work that arrives+any other way (a copilot turn off a queue, a graph node run by the executor)+starts its own here, so each unit of work is a bounded transaction with a+stable, low-cardinality name that can be compared release over release:++* ``start_transaction`` runs a block as one transaction.+* ``set_timing`` / ``set_data`` put numbers on it as span data, which Sentry's+ span explorer aggregates as a typed attribute, e.g.+ ``p95(tags[copilot.turn.ttft_ms,number])``.+* ``suppress_child_spans`` keeps I/O that repeats for as long as a unit of work+ lasts (stream polls, lock keepalives) from filling the SDK's span limit.+* ``trace_propagation_headers`` carries the active trace across a handover that+ is not an HTTP request, such as a queue message.++Sentry's server-side data scrubbing drops span data whose name contains+``token``, ``auth``, ``secret`` or ``password``, so names must avoid them:+``ttft_ms``, not ``first_token_ms``.++Nothing here raises into the caller. When Sentry is off or a call fails, the+work runs untraced.+"""++import asyncio+import hashlib+import logging+import uuid+from collections.abc import Iterator, Mapping+from contextlib import contextmanager+from typing import Any+from urllib.parse import urlsplit++import sentry_sdk+from sentry_sdk.tracing import NoOpSpan, Span, Transaction, TransactionSource++logger = logging.getLogger(__name__)++SENTRY_TRACE_HEADER = "sentry-trace"+BAGGAGE_HEADER = "baggage"++# The probe and scrape endpoints our services serve, matched by exact path: a+# route that merely ends in one of these names (a store agent slugged+# "health") is real traffic.+_UNTRACED_PATHS = frozenset(+ {+ # WebSocket server: Kubernetes probe.+ "/",+ # Every FastAPI service: Prometheus scrape.+ "/metrics",+ # REST API and WebSocket server: Kubernetes probe, load balancer check.+ "/health",+ # Every RPC service (AppService): Kubernetes probe.+ "/health_check",+ "/health_check_async",+ # Chat service health check.+ "/api/chat/health",+ # Codex compatibility gateway.+ "/healthz",+ }+)+++def traces_sampler(sampling_context: Mapping[str, Any]) -> float:+ """Trace everything except probes, metric scrapes and WebSocket connections.++ A WebSocket transaction lasts as long as its connection, so it is neither+ bounded nor comparable; probes and scrapes are frequent and say nothing.+ Any other transaction keeps its parent's decision when it has one.+ """+ try:+ if _is_untraced(sampling_context):+ return 0.0+ parent_sampled = sampling_context.get("parent_sampled")+ if parent_sampled is not None:+ return 1.0 if parent_sampled else 0.0+ except Exception:+ logger.debug("traces_sampler could not classify a transaction", exc_info=True)+ return 1.0+++def _is_untraced(sampling_context: Mapping[str, Any]) -> bool:+ transaction = sampling_context.get("transaction_context") or {}+ asgi_scope = sampling_context.get("asgi_scope") or {}+ if (+ transaction.get("op") == "websocket.server"+ or asgi_scope.get("type") == "websocket"+ ):+ return True+ return _request_path(sampling_context) in _UNTRACED_PATHS+++def _request_path(sampling_context: Mapping[str, Any]) -> str | None:+ """The request's path, without scheme, host or query string.++ It comes from the ASGI scope or the aiohttp request when the integration+ passes one, or else from the transaction name, which at sampling time is+ still the raw URL.+ """+ path = (sampling_context.get("asgi_scope") or {}).get("path")+ if isinstance(path, str):+ return path+ path = getattr(sampling_context.get("aiohttp_request"), "path", None)+ if isinstance(path, str):+ return path+ transaction = sampling_context.get("transaction_context") or {}+ name = transaction.get("name")+ if transaction.get("op") != "http.server" or not isinstance(name, str):+ return None+ return urlsplit(name).path+++@contextmanager+def start_transaction(+ op: str,+ name: str,+ *,+ trace_headers: Mapping[str, str] | None = None,+ trace_id: str | None = None,+ tags: Mapping[str, str] | None = None,+ data: Mapping[str, Any] | None = None,+) -> Iterator[Span]:+ """Run the block as its own transaction and yield it.++ ``op`` and ``name`` must be low-cardinality; ids belong in ``data``. The+ trace is, in order: the one ``trace_headers`` carries (work handed over+ from elsewhere), the transaction already running in this scope (the block+ becomes its child), ``trace_id`` (units of work that share a trace by key),+ or a new one.++ The block runs in a forked Sentry scope, so neither the transaction nor+ anything the block sets on the scope reaches concurrent work that shares+ the caller's. An exception leaving the block marks the transaction failed,+ or ``cancelled`` for a cancellation, and propagates unchanged; a block that+ returns without setting a status leaves it ``ok``.+ """+ if not _tracing_active():+ yield NoOpSpan()+ return+ with sentry_sdk.isolation_scope():+ span = _begin_transaction(op, name, trace_headers, trace_id, tags, data)+ if span is None:+ yield NoOpSpan()+ return+ try:+ yield span+ except BaseException as exc:+ set_status(_status_for(exc), span=span)+ raise+ else:+ if span.status is None:+ set_status("ok", span=span)+ finally:+ _finish(span)+++def _begin_transaction(+ op: str,+ name: str,+ trace_headers: Mapping[str, str] | None,+ trace_id: str | None,+ tags: Mapping[str, str] | None,+ data: Mapping[str, Any] | None,+) -> Span | None:+ """The started and entered transaction, or None if it could not start."""+ try:+ transaction = sentry_sdk.continue_trace(+ _incoming_trace(trace_headers, trace_id),+ op=op,+ name=name,+ source=TransactionSource.TASK,+ )+ for key, value in (tags or {}).items():+ transaction.set_tag(key, value)+ for key, value in (data or {}).items():+ transaction.set_data(key, value)+ span = sentry_sdk.start_transaction(transaction)+ span.__enter__()+ return span+ except Exception:+ logger.warning("Could not start Sentry transaction %s", name, exc_info=True)+ return None+++def _incoming_trace(+ trace_headers: Mapping[str, str] | None, trace_id: str | None+) -> dict[str, str]:+ if trace_headers and trace_headers.get(SENTRY_TRACE_HEADER):+ return {+ key: value+ for key, value in trace_headers.items()+ if key in (SENTRY_TRACE_HEADER, BAGGAGE_HEADER)+ }+ if active := trace_propagation_headers():+ return active+ if trace_id:+ return {SENTRY_TRACE_HEADER: trace_id}+ return {}+++def _finish(span: Span) -> None:+ try:+ span.__exit__(None, None, None)+ except Exception:+ logger.warning("Could not finish Sentry transaction", exc_info=True)+++def _status_for(exc: BaseException) -> str:+ if isinstance(exc, (asyncio.CancelledError, GeneratorExit)):+ return "cancelled"+ if isinstance(exc, (KeyboardInterrupt, SystemExit)):+ return "aborted"+ return "internal_error"+++def set_status(status: str, *, span: Span | None = None) -> None:+ """Set the status of ``span`` or else the current transaction, from+ Sentry's vocabulary (``ok``, ``cancelled``, ``internal_error``,+ ``invalid_argument``, ...)."""+ try:+ target = span if span is not None else current_transaction()+ if target is not None:+ target.set_status(status)+ except Exception:+ logger.debug("Could not set Sentry span status", exc_info=True)+++def current_transaction() -> Transaction | None:+ """The transaction running in the current scope, if any."""+ try:+ return sentry_sdk.get_current_scope().transaction+ except Exception:+ return None+++def set_timing(name: str, milliseconds: float, *, span: Span | None = None) -> None:+ """Record a duration in milliseconds, named ``<area>.<what>_ms``.++ It lands on ``span`` or else the current transaction, as numeric span data+ that the span explorer can aggregate. Negative durations, which only clock+ skew between hosts produces, are recorded as zero.+ """+ try:+ value = round(max(float(milliseconds), 0.0), 3)+ except (TypeError, ValueError):+ logger.debug("Not a duration for %s: %r", name, milliseconds)+ return+ set_data(name, value, span=span)+++def set_data(name: str, value: Any, *, span: Span | None = None) -> None:+ """Attach a value to ``span`` or else the current transaction."""+ try:+ target = span if span is not None else current_transaction()+ if target is not None:+ target.set_data(name, value)+ except Exception:+ logger.debug("Could not set Sentry span data %s", name, exc_info=True)+++@contextmanager+def suppress_child_spans() -> Iterator[None]:+ """Run the block without recording its spans on the active transaction.++ For I/O that repeats for as long as a unit of work lasts. The block's spans+ are dropped, not its errors: anything it captures is still reported, on the+ same trace.+ """+ if not _tracing_active() or not _span_active():+ yield+ return+ try:+ scope_manager = sentry_sdk.new_scope()+ scope = scope_manager.__enter__()+ scope.span = None+ except Exception:+ logger.debug("Could not suppress Sentry child spans", exc_info=True)+ yield+ return+ try:+ yield+ finally:+ scope_manager.__exit__(None, None, None)+++def trace_propagation_headers() -> dict[str, str]:+ """The active transaction's ``sentry-trace`` and ``baggage`` headers.++ Empty outside a transaction, so a handover from untraced code starts a new+ trace instead of joining whichever trace its thread last saw.+ """+ try:+ if not _tracing_active():+ return {}+ span = sentry_sdk.get_current_scope().span+ if span is None or span.containing_transaction is None:+ return {}+ return dict(span.iter_headers())+ except Exception:+ logger.debug("Could not read Sentry trace headers", exc_info=True)+ return {}+++def trace_id_for(key: str) -> str:+ """A trace id derived from ``key``, for units of work that share a trace.++ A UUID maps to its own 32 hex digits, so such a trace can be opened+ straight from the id it was keyed by.+ """+ try:+ return uuid.UUID(key).hex+ except (AttributeError, TypeError, ValueError):+ return hashlib.sha256(str(key).encode()).hexdigest()[:32]+++def _tracing_active() -> bool:+ try:+ return sentry_sdk.get_client().is_active()+ except Exception:+ return False+++def _span_active() -> bool:+ """Whether a span started now would get a parent, as ``Scope.start_span``+ decides it."""+ try:+ return (+ sentry_sdk.get_current_scope().span is not None+ or sentry_sdk.get_isolation_scope().span is not None+ )+ except Exception:+ return Truediff --git a/autogpt_platform/backend/backend/util/tracing_test.py b/autogpt_platform/backend/backend/util/tracing_test.pynew file mode 100644index 0000000000..09ab9c5b65--- /dev/null+++ b/autogpt_platform/backend/backend/util/tracing_test.py@@ -0,0 +1,495 @@+"""Tests for the Sentry tracing convention in ``backend.util.tracing``.++Most run against a live SDK client (the ``sentry_transactions`` fixture) and+assert on the transaction payloads it would have sent, so they pin what lands+in Sentry rather than which SDK functions were called.+"""++import asyncio+import uuid+from types import SimpleNamespace+from typing import Any++import pytest+import sentry_sdk+from fastapi import FastAPI+from fastapi.testclient import TestClient+from sentry_sdk.integrations.asgi import SentryAsgiMiddleware+from sentry_sdk.tracing import NoOpSpan++from backend.util import tracing++# ---------- traces_sampler ----------+++def _http(path: str, *, parent_sampled: bool | None = None) -> dict:+ return {+ "transaction_context": {"op": "http.server", "name": path},+ "parent_sampled": parent_sampled,+ "asgi_scope": {"type": "http", "path": path},+ }+++ "path",+ [+ "/metrics",+ "/",+ "/health",+ "/health_check",+ "/health_check_async",+ "/api/chat/health",+ "/healthz",+ ],+)+def test_probes_and_scrapes_are_not_traced(path: str) -> None:+ assert tracing.traces_sampler(_http(path)) == 0.0+++def test_a_websocket_connection_is_not_traced() -> None:+ context = {+ "transaction_context": {"op": "websocket.server", "name": "/ws"},+ "parent_sampled": None,+ "asgi_scope": {"type": "websocket", "path": "/ws"},+ }++ assert tracing.traces_sampler(context) == 0.0+++def test_a_websocket_is_recognised_by_op_without_an_asgi_scope() -> None:+ context = {"transaction_context": {"op": "websocket.server", "name": "/ws"}}++ assert tracing.traces_sampler(context) == 0.0+++ "name",+ ["http://10.0.0.5:8006/metrics", "http://10.0.0.5:8006/health?probe=1"],+)+def test_a_probe_named_after_its_full_url_is_not_traced(name: str) -> None:+ """Before routing, an HTTP transaction is named after its raw URL."""+ context = {+ "transaction_context": {"op": "http.server", "name": name},+ "parent_sampled": None,+ }++ assert tracing.traces_sampler(context) == 0.0+++def test_a_probe_on_an_aiohttp_server_is_not_traced() -> None:+ """aiohttp names a transaction generically until its route resolves; the+ request it hands the sampler has the path."""+ context = {+ "transaction_context": {+ "op": "http.server",+ "name": "generic AIOHTTP request",+ },+ "parent_sampled": None,+ "aiohttp_request": SimpleNamespace(path="/healthz"),+ }++ assert tracing.traces_sampler(context) == 0.0+ context["aiohttp_request"] = SimpleNamespace(path="/v1/messages")+ assert tracing.traces_sampler(context) == 1.0+++def test_a_dropped_endpoint_stays_dropped_even_with_a_sampled_parent() -> None:+ assert tracing.traces_sampler(_http("/metrics", parent_sampled=True)) == 0.0+++ "path",+ [+ "/api/store/agents/alice/health",+ "/api/store/agents/alice/readyz",+ "/api/store/agents/alice/healthz",+ "/api/store/agents/alice/livez",+ "/api/store/agents/alice/readiness",+ "/api/store/agents/alice/health_check",+ "/api/store/agents/alice/metrics",+ "/health/",+ ],+)[email protected]("parent_sampled", [None, True])+def test_a_route_that_only_ends_like_a_probe_is_traced(+ path: str, parent_sampled: bool | None+) -> None:+ """Only the exact probe paths are dropped: a store agent slugged "health"+ is real traffic."""+ context = _http(path, parent_sampled=parent_sampled)+ url_named = {+ "transaction_context": {+ "op": "http.server",+ "name": f"http://10.0.0.5:8006{path}",+ },+ "parent_sampled": parent_sampled,+ }++ assert tracing.traces_sampler(context) == 1.0+ assert tracing.traces_sampler(url_named) == 1.0+++def test_through_the_asgi_middleware_only_the_probe_is_dropped(+ sentry_transactions,+) -> None:+ """End to end through the SDK's ASGI middleware, as each service runs."""+ app = FastAPI()++ @app.get("/health")+ def health() -> dict:+ return {"status": "healthy"}++ @app.get("/api/store/agents/{username}/{agent_name}")+ def agent(username: str, agent_name: str) -> dict:+ return {"agent": agent_name}++ client = TestClient(SentryAsgiMiddleware(app, transaction_style="url"))+ for path in (+ "/health",+ "/api/store/agents/alice/health",+ "/api/store/agents/alice/readyz",+ ):+ assert client.get(path).status_code == 200++ assert [t["request"]["url"] for t in sentry_transactions] == [+ "http://testserver/api/store/agents/alice/health",+ "http://testserver/api/store/agents/alice/readyz",+ ]+ assert {t["transaction"] for t in sentry_transactions} == {+ "/api/store/agents/{username}/{agent_name}"+ }++[email protected]("parent_sampled, expected", [(True, 1.0), (False, 0.0)])+def test_the_parents_decision_is_kept(parent_sampled: bool, expected: float) -> None:+ context = _http("/api/graphs", parent_sampled=parent_sampled)++ assert tracing.traces_sampler(context) == expected+++ "context",+ [+ _http("/api/chat/sessions/{session_id}/stream"),+ _http("/update_node_execution_status"),+ {+ "transaction_context": {+ "op": "executor.node",+ "name": "AITextGeneratorBlock",+ }+ },+ {"transaction_context": {"op": "copilot.turn", "name": "copilot.turn"}},+ ],+)+def test_everything_else_is_traced(context: dict) -> None:+ assert tracing.traces_sampler(context) == 1.0+++def test_a_malformed_sampling_context_is_traced_rather_than_raising() -> None:+ assert tracing.traces_sampler({"transaction_context": "not-a-dict"}) == 1.0+++# ---------- start_transaction ----------+++def _trace(transaction: dict) -> dict:+ return transaction["contexts"]["trace"]+++def test_a_unit_of_work_becomes_one_transaction(sentry_transactions) -> None:+ with tracing.start_transaction(+ "copilot.turn",+ "copilot.turn",+ tags={"copilot.engine": "sdk"},+ data={"copilot.session_id": "s-1"},+ ):+ with sentry_sdk.start_span(op="db.redis", name="GET"):+ pass++ [transaction] = sentry_transactions+ assert transaction["transaction"] == "copilot.turn"+ assert transaction["transaction_info"] == {"source": "task"}+ assert _trace(transaction)["op"] == "copilot.turn"+ assert _trace(transaction)["status"] == "ok"+ assert _trace(transaction)["data"]["copilot.session_id"] == "s-1"+ assert transaction["tags"]["copilot.engine"] == "sdk"+ assert [span["op"] for span in transaction["spans"]] == ["db.redis"]+++def test_handed_over_trace_headers_are_continued(sentry_transactions) -> None:+ trace_id, parent_span_id = "a" * 32, "b" * 16++ with tracing.start_transaction(+ "copilot.turn",+ "copilot.turn",+ trace_headers={"sentry-trace": f"{trace_id}-{parent_span_id}-1"},+ ):+ pass++ [transaction] = sentry_transactions+ assert _trace(transaction)["trace_id"] == trace_id+ assert _trace(transaction)["parent_span_id"] == parent_span_id+++def test_an_unsampled_parent_is_not_traced(sentry_transactions) -> None:+ with tracing.start_transaction(+ "copilot.turn",+ "copilot.turn",+ trace_headers={"sentry-trace": f"{'a' * 32}-{'b' * 16}-0"},+ ):+ pass++ assert sentry_transactions == []+++def test_a_keyed_unit_joins_the_trace_for_its_key(sentry_transactions) -> None:+ trace_id = tracing.trace_id_for(str(uuid.uuid4()))++ for _ in range(2):+ with tracing.start_transaction("executor.node", "Block", trace_id=trace_id):+ pass++ assert [_trace(t)["trace_id"] for t in sentry_transactions] == [trace_id] * 2+ assert all(_trace(t).get("parent_span_id") is None for t in sentry_transactions)+++def test_a_unit_inside_another_becomes_its_child(sentry_transactions) -> None:+ with tracing.start_transaction("copilot.turn", "copilot.turn") as turn:+ with tracing.start_transaction("copilot.tool", "run_block"):+ with sentry_sdk.start_span(op="http.client", name="POST /rpc"):+ pass+ with sentry_sdk.start_span(op="db.redis", name="XADD"):+ pass++ tool, outer = sentry_transactions+ assert tool["transaction"] == "run_block"+ assert _trace(tool)["trace_id"] == turn.trace_id+ assert _trace(tool)["parent_span_id"] == turn.span_id+ assert [span["op"] for span in tool["spans"]] == ["http.client"]+ assert [span["op"] for span in outer["spans"]] == ["db.redis"]+++def test_handed_over_headers_win_over_the_running_transaction(+ sentry_transactions,+) -> None:+ trace_id = "c" * 32++ with tracing.start_transaction("copilot.turn", "copilot.turn"):+ with tracing.start_transaction(+ "copilot.turn",+ "copilot.turn",+ trace_headers={"sentry-trace": f"{trace_id}-{'d' * 16}-1"},+ ):+ pass++ inner, _ = sentry_transactions+ assert _trace(inner)["trace_id"] == trace_id+++def test_the_caller_scope_is_left_as_it_was(sentry_transactions) -> None:+ scope = sentry_sdk.get_current_scope()+ before = scope.span++ with tracing.start_transaction("executor.node", "Block"):+ sentry_sdk.set_tag("executor.leaked", "yes")++ assert sentry_sdk.get_current_scope().span is before+ with tracing.start_transaction("executor.node", "Block"):+ pass+ assert "executor.leaked" not in sentry_transactions[-1].get("tags", {})+++async def test_concurrent_units_sharing_a_scope_keep_their_own_spans(+ sentry_transactions,+) -> None:+ """Coroutines handed to another loop with run_coroutine_threadsafe share+ one Sentry scope; each unit's spans must still land on its own transaction."""+ both_started = asyncio.Event()+ started = 0++ async def unit(name: str) -> None:+ nonlocal started+ with tracing.start_transaction("executor.node", name):+ started += 1+ if started == 2:+ both_started.set()+ await both_started.wait()+ with sentry_sdk.start_span(op="http.client", name=f"call from {name}"):+ await asyncio.sleep(0)++ await asyncio.gather(unit("A"), unit("B"))++ by_name = {t["transaction"]: t for t in sentry_transactions}+ assert [s["description"] for s in by_name["A"]["spans"]] == ["call from A"]+ assert [s["description"] for s in by_name["B"]["spans"]] == ["call from B"]+++def test_an_exception_fails_the_transaction_and_propagates(+ sentry_transactions,+) -> None:+ with pytest.raises(RuntimeError, match="boom"):+ with tracing.start_transaction("executor.node", "Block"):+ raise RuntimeError("boom")++ [transaction] = sentry_transactions+ assert _trace(transaction)["status"] == "internal_error"+++async def test_a_cancellation_is_recorded_as_cancelled(sentry_transactions) -> None:+ async def turn() -> None:+ with tracing.start_transaction("copilot.turn", "copilot.turn"):+ await asyncio.sleep(10)++ task = asyncio.create_task(turn())+ await asyncio.sleep(0)+ task.cancel()+ with pytest.raises(asyncio.CancelledError):+ await task++ [transaction] = sentry_transactions+ assert _trace(transaction)["status"] == "cancelled"+++def test_a_status_set_by_the_block_is_kept(sentry_transactions) -> None:+ with tracing.start_transaction("executor.node", "Block"):+ with sentry_sdk.start_span(op="function", name="inner"):+ tracing.set_status("invalid_argument")++ assert _trace(sentry_transactions[0])["status"] == "invalid_argument"+++def test_without_a_sentry_client_the_block_runs_untraced() -> None:+ assert not sentry_sdk.get_client().is_active()+ ran = False++ with tracing.start_transaction("executor.node", "Block") as span:+ ran = True++ assert ran+ assert isinstance(span, NoOpSpan)+++def test_a_transaction_that_cannot_start_leaves_the_block_running(+ sentry_transactions, monkeypatch+) -> None:+ def refuse(*args, **kwargs):+ raise RuntimeError("sdk failure")++ monkeypatch.setattr(sentry_sdk, "continue_trace", refuse)+ ran = False++ with tracing.start_transaction("executor.node", "Block") as span:+ ran = True++ assert ran+ assert isinstance(span, NoOpSpan)+ assert sentry_transactions == []+++# ---------- timings and data ----------+++def test_timings_land_on_the_current_transaction(sentry_transactions) -> None:+ with tracing.start_transaction("copilot.turn", "copilot.turn"):+ with sentry_sdk.start_span(op="function", name="inner"):+ tracing.set_timing("copilot.turn.ttft_ms", 1234.56789)+ tracing.set_data("copilot.turn.chunk_count", 7)++ data = _trace(sentry_transactions[0])["data"]+ assert data["copilot.turn.ttft_ms"] == 1234.568+ assert data["copilot.turn.chunk_count"] == 7+++def test_a_negative_duration_is_recorded_as_zero(sentry_transactions) -> None:+ with tracing.start_transaction("executor.node", "Block") as span:+ tracing.set_timing("executor.node.queue_wait_ms", -3.0, span=span)++ assert _trace(sentry_transactions[0])["data"]["executor.node.queue_wait_ms"] == 0+++def test_a_value_that_is_not_a_duration_is_ignored(sentry_transactions) -> None:+ not_a_duration: Any = "soon"++ with tracing.start_transaction("copilot.turn", "copilot.turn"):+ tracing.set_timing("copilot.turn.ttft_ms", not_a_duration)++ assert "copilot.turn.ttft_ms" not in _trace(sentry_transactions[0])["data"]+++def test_a_timing_outside_a_transaction_is_ignored(sentry_transactions) -> None:+ tracing.set_timing("copilot.turn.ttft_ms", 10.0)++ assert sentry_transactions == []+++# ---------- suppress_child_spans ----------+++def test_suppressed_spans_are_not_recorded(sentry_transactions) -> None:+ with tracing.start_transaction("copilot.turn", "copilot.turn"):+ with tracing.suppress_child_spans():+ for _ in range(5):+ with sentry_sdk.start_span(op="db.redis", name="XREAD"):+ pass+ with sentry_sdk.start_span(op="db.redis", name="HGETALL"):+ pass++ [transaction] = sentry_transactions+ assert [span["description"] for span in transaction["spans"]] == ["HGETALL"]+++def test_suppression_keeps_errors_on_the_transactions_trace(+ sentry_transactions,+) -> None:+ with tracing.start_transaction("copilot.turn", "copilot.turn") as span:+ with tracing.suppress_child_spans():+ trace_context = sentry_sdk.get_current_scope().get_trace_context()++ assert trace_context["trace_id"] == span.trace_id+++def test_suppression_without_a_transaction_is_harmless(sentry_transactions) -> None:+ with tracing.suppress_child_spans():+ with sentry_sdk.start_span(op="db.redis", name="GET"):+ pass++ assert sentry_transactions == []+++# ---------- trace_propagation_headers ----------+++def test_no_headers_are_handed_over_outside_a_transaction(+ sentry_transactions,+) -> None:+ assert tracing.trace_propagation_headers() == {}+++def test_headers_carry_the_active_trace(sentry_transactions) -> None:+ with tracing.start_transaction("copilot.turn", "copilot.turn") as span:+ headers = tracing.trace_propagation_headers()++ assert headers["sentry-trace"] == f"{span.trace_id}-{span.span_id}-1"+ assert f"sentry-trace_id={span.trace_id}" in headers["baggage"]+++def test_no_headers_without_a_sentry_client() -> None:+ assert tracing.trace_propagation_headers() == {}+++# ---------- trace_id_for ----------+++def test_a_uuid_key_is_its_own_trace_id() -> None:+ key = uuid.uuid4()++ assert tracing.trace_id_for(str(key)) == key.hex+++def test_any_other_key_maps_to_a_stable_trace_id() -> None:+ trace_id = tracing.trace_id_for("graph-exec-1")++ assert trace_id == tracing.trace_id_for("graph-exec-1")+ assert trace_id != tracing.trace_id_for("graph-exec-2")+ assert len(trace_id) == 32+ int(trace_id, 16)