AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
feat(backend/copilot): group each AutoPilot reply's model calls into one PostHog trace (SECRT-2979) (#15604)
Krzysztof Czerwinski45e6f2281f71
diff --git a/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py b/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.pyindex b509950cb1..825fc04507 100644--- a/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py+++ b/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py@@ -37,7 +37,7 @@ from backend.copilot.baseline.service import ()from backend.copilot.context import get_execution_context, set_execution_contextfrom backend.copilot.expert_context import ExpertSessionUnavailableError-from backend.copilot.llm_attribution import set_turn_impersonated+from backend.copilot.llm_attribution import set_turn_impersonated, set_turn_trace_idfrom backend.copilot.llm_attribution import settings as attribution_settingsfrom backend.copilot.model import ChatMessage, ChatSessionfrom backend.copilot.model_router import ResolvedModel@@ -1237,6 +1237,7 @@ class TestBaselineCostExtraction:),):set_turn_impersonated(True)+ set_turn_trace_id("turn-1")try:await _baseline_llm_caller(messages=[{"role": "user", "content": "hi"}],@@ -1245,6 +1246,7 @@ class TestBaselineCostExtraction:)finally:set_turn_impersonated(False)+ set_turn_trace_id(None)assert create_mock.await_args is not Noneextra_body = create_mock.await_args.kwargs["extra_body"]@@ -1254,6 +1256,7 @@ class TestBaselineCostExtraction:"generation_name": "copilot-baseline","environment": attribution_settings.config.app_env.value,"impersonated": True,+ "trace_id": "turn-1",}@pytest.mark.asynciodiff --git a/autogpt_platform/backend/backend/copilot/executor/processor.py b/autogpt_platform/backend/backend/copilot/executor/processor.pyindex 75676b9e7c..482e685cf8 100644--- a/autogpt_platform/backend/backend/copilot/executor/processor.py+++ b/autogpt_platform/backend/backend/copilot/executor/processor.py@@ -32,7 +32,7 @@ from backend.copilot.expert_context import (EXPERT_SESSION_TEMPORARY_MESSAGE,ExpertSessionUnavailableError,)-from backend.copilot.llm_attribution import set_turn_impersonated+from backend.copilot.llm_attribution import set_turn_impersonated, set_turn_trace_idfrom backend.copilot.model import maybe_append_user_messagefrom backend.copilot.rate_limit import RateLimitUnavailablefrom backend.copilot.response_model import (@@ -837,6 +837,7 @@ class CoPilotProcessor:envelope = taint_for_source_platform(entry.envelope, session)set_turn_unattended(session, scheduled=entry.unattended)set_turn_impersonated(entry.impersonated)+ set_turn_trace_id(entry.turn_id)set_turn_credential_pins(entry.credential_pins)turn.engine_started(engine)raw_stream = stream_fn(diff --git a/autogpt_platform/backend/backend/copilot/executor/processor_test.py b/autogpt_platform/backend/backend/copilot/executor/processor_test.pyindex 0f9fb0d1ed..5dd7017352 100644--- a/autogpt_platform/backend/backend/copilot/executor/processor_test.py+++ b/autogpt_platform/backend/backend/copilot/executor/processor_test.py@@ -33,7 +33,7 @@ from backend.copilot.expert_context import (EXPERT_SESSION_TEMPORARY_MESSAGE,ExpertSessionUnavailableError,)-from backend.copilot.llm_attribution import is_turn_impersonated+from backend.copilot.llm_attribution import is_turn_impersonated, turn_trace_idfrom backend.copilot.model import ChatSessionfrom backend.copilot.rate_limit import UserPaywalledErrorfrom backend.copilot.response_model import (@@ -233,6 +233,7 @@ class TestExecuteAsyncScheduledTurnContext:seen["unattended"] = is_unattended_turn()seen["pins"] = dict(turn_credential_pins())seen["impersonated"] = is_turn_impersonated()+ seen["trace_id"] = turn_trace_id()return MagicMock()with (@@ -275,13 +276,23 @@ class TestExecuteAsyncScheduledTurnContext:seen = await self._seen_by_the_turn(entry)- assert seen == {"unattended": True, "pins": {"exa": pin}, "impersonated": False}+ assert seen == {+ "unattended": True,+ "pins": {"exa": pin},+ "impersonated": False,+ "trace_id": "turn-1",+ }@pytest.mark.asyncioasync def test_a_typed_turn_is_watched_and_has_no_pins(self) -> None:seen = await self._seen_by_the_turn(_make_entry())- assert seen == {"unattended": False, "pins": {}, "impersonated": False}+ assert seen == {+ "unattended": False,+ "pins": {},+ "impersonated": False,+ "trace_id": "turn-1",+ }@pytest.mark.asyncioasync def test_an_impersonated_turn_is_marked_for_llm_analytics(self) -> None:@@ -291,6 +302,15 @@ class TestExecuteAsyncScheduledTurnContext:assert seen["impersonated"] is True+ @pytest.mark.asyncio+ async def test_the_turn_id_is_the_trace_of_its_llm_calls(self) -> None:+ """Every model call of one reply groups into one PostHog trace."""+ entry = _make_entry().model_copy(update={"turn_id": "turn-42"})++ seen = await self._seen_by_the_turn(entry)++ assert seen["trace_id"] == "turn-42"+class TestExecuteAsyncAclose:"""``_execute_async`` must call ``aclose`` on the published stream bothdiff --git a/autogpt_platform/backend/backend/copilot/feedback.py b/autogpt_platform/backend/backend/copilot/feedback.pyindex 10400ccd27..e088473864 100644--- a/autogpt_platform/backend/backend/copilot/feedback.py+++ b/autogpt_platform/backend/backend/copilot/feedback.py@@ -20,9 +20,10 @@ consent, like the generations OpenRouter Broadcast sends there.``$ai_session_id`` is the chat session id, the same value the chat's``$ai_generation`` events carry, which is what joins a rating to thegenerations of its chat; ``message_id`` names the reply. It carries no-``$ai_trace_id``: Broadcast assigns the generations' trace ids, so we have-none to match. Rating again sends another event; the latest per-``feedback_id`` is the rating that stands.+``$ai_trace_id`` yet: each generation's trace id is the turn that made it (see+``llm_attribution``), but a reply row does not record which turn wrote it.+Rating again sends another event; the latest per ``feedback_id`` is the+rating that stands."""import loggingdiff --git a/autogpt_platform/backend/backend/copilot/llm_attribution.py b/autogpt_platform/backend/backend/copilot/llm_attribution.pyindex 1f555493e8..8503ecf6a7 100644--- a/autogpt_platform/backend/backend/copilot/llm_attribution.py+++ b/autogpt_platform/backend/backend/copilot/llm_attribution.py@@ -8,9 +8,15 @@ OpenRouter Broadcast sends every generation to PostHog LLM analytics as anthe distinct id every other backend and browser event uses, so thegeneration lands on the same person as their product events and replays.- ``session_id`` becomes ``$ai_session_id``: the chat session id.+- ``trace.trace_id`` becomes ``$ai_trace_id``: the id of the turn that made+ the call, so every model call behind one AutoPilot reply (the main loop,+ sub-agents, compaction, the chat title) groups into one trace.- ``trace.generation_name`` becomes ``$ai_span_name``; every other ``trace``key arrives as ``metadata_<key>`` (``metadata_environment``,- ``metadata_impersonated``).+ ``metadata_impersonated``, and ``metadata_revision``: the deployed build's+ git sha from ``config.sentry_release``, left out when that is blank).+ Broadcast sends these events itself, not through our PostHog client, so+ they never get a property the client adds to every event it captures.Only calls that go through OpenRouter are covered. Direct Anthropic, ClaudeCode subscription and user-connected (Codex) routes never pass OpenRouter, so@@ -37,6 +43,11 @@ GenerationName = Literal["copilot-title", "copilot-sdk", "copilot-baseline"]# executor sets it for every turn, so a value never carries over._turn_impersonated: ContextVar[bool] = ContextVar("_turn_impersonated", default=False)+# The running turn's id, sent as the trace id of every call it makes. The+# executor sets it for every turn, so a value never carries over; a call made+# outside a turn sends none.+_turn_trace_id: ContextVar[str | None] = ContextVar("_turn_trace_id", default=None)+def openrouter_attribution(*,@@ -52,6 +63,12 @@ def openrouter_attribution("impersonated": is_turn_impersonated(),}}+ revision = settings.config.sentry_release.strip()+ if revision:+ body["trace"]["revision"] = revision+ trace_id = turn_trace_id()+ if trace_id:+ body["trace"]["trace_id"] = trace_id[:_MAX_ID_CHARS]if user_id:body["user"] = user_id[:_MAX_ID_CHARS]if session_id:@@ -68,6 +85,15 @@ def is_turn_impersonated() -> bool:return _turn_impersonated.get()+def set_turn_trace_id(turn_id: str | None) -> None:+ """Make *turn_id* the trace of every model call the running turn makes."""+ _turn_trace_id.set(turn_id or None)+++def turn_trace_id() -> str | None:+ return _turn_trace_id.get()++def is_impersonated_request(request: Request) -> bool:"""Whether an admin made *request* as another user.diff --git a/autogpt_platform/backend/backend/copilot/llm_attribution_test.py b/autogpt_platform/backend/backend/copilot/llm_attribution_test.pyindex 02964b6cd6..0ba9d68a8d 100644--- a/autogpt_platform/backend/backend/copilot/llm_attribution_test.py+++ b/autogpt_platform/backend/backend/copilot/llm_attribution_test.py@@ -11,15 +11,19 @@ from backend.copilot.llm_attribution import (is_turn_impersonated,openrouter_attribution,set_turn_impersonated,+ set_turn_trace_id,+ turn_trace_id,)from backend.util.settings import AppEnvironment@pytest.fixture(autouse=True)-def _not_impersonated():+def _outside_a_turn():set_turn_impersonated(False)+ set_turn_trace_id(None)yieldset_turn_impersonated(False)+ set_turn_trace_id(None)def test_attribution_names_the_user_chat_and_environment(mocker):@@ -51,6 +55,25 @@ def test_attribution_without_a_user_or_chat_still_tags_the_environment():assert body["trace"]["generation_name"] == "copilot-baseline"+ ("sentry_release", "expected"),+ [("0123abcd\n", {"revision": "0123abcd"}), ("", {})],+ ids=["revision set", "no revision"],+)+def test_attribution_names_the_revision_that_made_the_call(+ mocker, sentry_release: str, expected: dict[str, str]+):+ mocker.patch.object(+ llm_attribution.settings.config, "sentry_release", sentry_release+ )++ body = openrouter_attribution(+ user_id=None, session_id=None, generation_name="copilot-sdk"+ )++ assert {k: v for k, v in body["trace"].items() if k == "revision"} == expected++def test_attribution_marks_an_impersonated_turn():set_turn_impersonated(True)@@ -75,6 +98,62 @@ async def test_impersonation_does_not_leak_into_another_turn():assert (admin, user) == (True, False)+def test_attribution_groups_a_turns_calls_under_its_trace():+ set_turn_trace_id("turn-1")++ body = openrouter_attribution(+ user_id="user-1", session_id="sess-1", generation_name="copilot-sdk"+ )++ assert body["trace"]["trace_id"] == "turn-1"+++def test_attribution_outside_a_turn_sends_no_trace_id():+ body = openrouter_attribution(+ user_id="user-1", session_id="sess-1", generation_name="copilot-title"+ )++ assert "trace_id" not in body["trace"]+++def test_a_blank_turn_id_sends_no_trace_id():+ set_turn_trace_id("")++ assert turn_trace_id() is None+ body = openrouter_attribution(+ user_id="user-1", session_id="sess-1", generation_name="copilot-sdk"+ )+ assert "trace_id" not in body["trace"]+++async def test_a_trace_id_does_not_leak_into_another_turn():+ async def turn(turn_id: str) -> str | None:+ set_turn_trace_id(turn_id)+ await asyncio.sleep(0)+ return openrouter_attribution(+ user_id="user-1", session_id="sess-1", generation_name="copilot-sdk"+ )["trace"]["trace_id"]++ first, second = await asyncio.gather(+ asyncio.create_task(turn("turn-1")), asyncio.create_task(turn("turn-2"))+ )++ assert (first, second) == ("turn-1", "turn-2")+++async def test_a_task_the_turn_starts_shares_its_trace():+ """The chat title is generated in a task the first turn starts, so its+ call lands in that turn's trace."""+ set_turn_trace_id("turn-1")++ async def title_call() -> str | None:+ return turn_trace_id()++ assert await asyncio.create_task(title_call()) == "turn-1"++@pytest.mark.parametrize(("headers", "expected"),[diff --git a/autogpt_platform/backend/backend/copilot/sdk/env_broadcast_test.py b/autogpt_platform/backend/backend/copilot/sdk/env_broadcast_test.pyindex 7094f4cf6a..fa3ad7dbd5 100644--- a/autogpt_platform/backend/backend/copilot/sdk/env_broadcast_test.py+++ b/autogpt_platform/backend/backend/copilot/sdk/env_broadcast_test.py@@ -11,7 +11,7 @@ from unittest.mock import patchimport pytestfrom backend.copilot.config import ChatConfig-from backend.copilot.llm_attribution import set_turn_impersonated+from backend.copilot.llm_attribution import set_turn_impersonated, set_turn_trace_idfrom backend.copilot.llm_attribution import settings as attribution_settingsfrom backend.copilot.sdk.env import build_sdk_env@@ -31,10 +31,12 @@ def _config(**overrides) -> ChatConfig:@pytest.fixture(autouse=True)-def _not_impersonated():+def _outside_a_turn():set_turn_impersonated(False)+ set_turn_trace_id(None)yieldset_turn_impersonated(False)+ set_turn_trace_id(None)def _extra_body(**kwargs) -> dict:@@ -73,6 +75,16 @@ def test_an_impersonated_turn_is_marked():assert body["trace"]["impersonated"] is True+def test_every_request_of_a_turn_carries_its_trace_id():+ """Broadcast maps ``trace.trace_id`` to ``$ai_trace_id``, so the turn's+ main, sub-agent and compaction calls group into one PostHog trace."""+ set_turn_trace_id("turn-1")++ body = _extra_body(session_id="sess-1", user_id="user-1")++ assert body["trace"]["trace_id"] == "turn-1"++def test_no_user_sends_no_user_fields():body = _extra_body(session_id="sess-1")diff --git a/autogpt_platform/backend/backend/copilot/service_unit_test.py b/autogpt_platform/backend/backend/copilot/service_unit_test.pyindex e9b66293ef..4593b1cdfc 100644--- a/autogpt_platform/backend/backend/copilot/service_unit_test.py+++ b/autogpt_platform/backend/backend/copilot/service_unit_test.py@@ -21,6 +21,7 @@ from openai.types.chat.chat_completion import Choicefrom openai.types.chat.chat_completion_message import ChatCompletionMessagefrom openai.types.completion_usage import CompletionUsage+from backend.copilot.llm_attribution import set_turn_trace_idfrom backend.copilot.service import (_TITLE_TIMEOUT_SECONDS,_fallback_title_from_message,@@ -674,6 +675,35 @@ class TestGenerateSessionTitle:"impersonated": False,}+ @pytest.mark.asyncio+ async def test_title_call_lands_in_the_trace_of_the_turn_that_asked(self):+ """The first turn generates the title, so the title call groups with+ that reply's other calls in PostHog."""+ resp = _build_completion(content="Title")+ helper = AsyncMock(return_value=resp)+ with (+ patch(+ "backend.copilot.service.call_provider_openai_compat_sync",+ new=helper,+ ),+ patch(+ "backend.copilot.service.config",+ MagicMock(+ aux_uses_openrouter=True,+ title_model="anthropic/claude-haiku",+ ),+ ),+ ):+ set_turn_trace_id("turn-1")+ try:+ await _generate_session_title(+ "hello world", user_id="user-abc", session_id="sess-abc"+ )+ finally:+ set_turn_trace_id(None)+ extra_body = helper.await_args.kwargs["extra_body"]+ assert extra_body["trace"]["trace_id"] == "turn-1"+@pytest.mark.asyncioasync def test_title_call_off_openrouter_carries_no_broadcast_fields(self):"""Broadcast only sees OpenRouter calls, and a direct-Anthropic auxdiff --git a/docs/engineering/tracking-plan.md b/docs/engineering/tracking-plan.mdindex c0e4112eac..db992cee10 100644--- a/docs/engineering/tracking-plan.md+++ b/docs/engineering/tracking-plan.md@@ -315,8 +315,8 @@ PostHog's own, not the analytics plan's.| Event | Sender | Status | Required properties | Fires when || --- | --- | --- | --- | --- |-| `$ai_generation` | OpenRouter Broadcast | live | set by OpenRouter from the request (`backend/copilot/llm_attribution.py`): `distinct_id` = the request's `user` (the platform user id), `$ai_session_id` = its `session_id` (the chat session id), `$ai_span_name` = `trace.generation_name` (`copilot-title`, `copilot-sdk`, `copilot-baseline`), `metadata_environment` (`local`, `dev`, `prod`), `metadata_impersonated`; model, tokens, cost and latency from OpenRouter | An AutoPilot LLM call goes through OpenRouter: a session title, an SDK turn (main agent, sub-agents and the CLI's compaction, via `CLAUDE_CODE_EXTRA_BODY`) or a baseline turn. |-| `$ai_metric` | backend | live | `$ai_metric_name` (`user-feedback`, `copy`), `$ai_metric_value` (`1` thumbs up or copy, `0` thumbs down), `$ai_session_id` and `chat_session_id` (the chat session id), `message_id` (the reply's row id), `message_sequence`, `feedback_id` (the stored rating, shared with its Langfuse score), `impersonated`; `$ai_feedback_text` when a thumbs-down has a comment | A reply is rated (`POST /api/chat/sessions/{id}/feedback`, `copilot/feedback.py`). Rating a reply again sends another event; the latest per `feedback_id` stands. |+| `$ai_generation` | OpenRouter Broadcast | live | set by OpenRouter from the request (`backend/copilot/llm_attribution.py`): `distinct_id` = the request's `user` (the platform user id), `$ai_session_id` = its `session_id` (the chat session id), `$ai_span_name` = `trace.generation_name` (`copilot-title`, `copilot-sdk`, `copilot-baseline`), `$ai_trace_id` = `trace.trace_id` (the id of the executor turn that made the call, so all calls behind one reply share it; absent for a call made outside a turn), `metadata_environment` (`local`, `dev`, `prod`), `metadata_impersonated`; `metadata_revision` (the deployed build's git sha, `config.sentry_release`) when it is set; model, tokens, cost and latency from OpenRouter | An AutoPilot LLM call goes through OpenRouter: a session title, an SDK turn (main agent, sub-agents and the CLI's compaction, via `CLAUDE_CODE_EXTRA_BODY`) or a baseline turn. |+| `$ai_metric` | backend | live | `$ai_metric_name` (`user-feedback`, `copy`), `$ai_metric_value` (`1` thumbs up or copy, `0` thumbs down), `$ai_session_id` and `chat_session_id` (the chat session id), `message_id` (the reply's row id), `message_sequence`, `feedback_id` (the stored rating, shared with its Langfuse score), `impersonated`; `$ai_feedback_text` when a thumbs-down has a comment. No `$ai_trace_id` (see below) | A reply is rated (`POST /api/chat/sessions/{id}/feedback`, `copilot/feedback.py`). Rating a reply again sends another event; the latest per `feedback_id` stands. |What joins them:@@ -325,11 +325,20 @@ What joins them:both land on the person with their product events and replays.- **Chat.** `$ai_session_id` is the chat session id on both, so a chat'sgenerations and its ratings group in the Sessions tab and join on it.- `message_id` says which reply was rated; a rating carries no- `$ai_trace_id`, since Broadcast assigns the generations' trace ids.+ `message_id` says which reply was rated. A rating carries no+ `$ai_trace_id` yet: a generation's trace id is the turn that made it, but a+ reply row does not record which turn wrote it, so a rating joins its+ reply's generations only through the chat.+- **Reply.** All generations behind one reply (the main loop, sub-agents,+ compaction and, on a chat's first reply, the title) share `$ai_trace_id`,+ the executor turn id, so PostHog groups them into one trace. An expert+ subtask runs as a turn of its own and gets its own trace.- **Environment.** OpenRouter forwards our `trace` keys as `metadata_<key>`,so a generation's environment is `metadata_environment`, not the`environment` base property; `$ai_metric` has both base properties as usual.+ Likewise the build is `metadata_revision` on a generation; `$ai_metric`+ sets no build property of its own and carries only the base properties our+ PostHog client adds to every event.- **Impersonation.** A turn or rating an admin made while impersonating theuser (`X-Act-As-User-Id`) is still sent, as the user, with`metadata_impersonated` / `impersonated` true, so it can be filtered out.