Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

feat(backend/copilot): group each AutoPilot reply's model calls into one PostHog trace (SECRT-2979) (#15604)

Krzysztof Czerwinski45e6f2281f71
diff --git a/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py b/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py
index b509950cb1..825fc04507 100644
--- a/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py
+++ b/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py
@@ -37,7 +37,7 @@ from backend.copilot.baseline.service import (
)
from backend.copilot.context import get_execution_context, set_execution_context
from backend.copilot.expert_context import ExpertSessionUnavailableError
-from backend.copilot.llm_attribution import set_turn_impersonated
+from backend.copilot.llm_attribution import set_turn_impersonated, set_turn_trace_id
from backend.copilot.llm_attribution import settings as attribution_settings
from backend.copilot.model import ChatMessage, ChatSession
from backend.copilot.model_router import ResolvedModel
@@ -1237,6 +1237,7 @@ class TestBaselineCostExtraction:
),
):
set_turn_impersonated(True)
+ set_turn_trace_id("turn-1")
try:
await _baseline_llm_caller(
messages=[{"role": "user", "content": "hi"}],
@@ -1245,6 +1246,7 @@ class TestBaselineCostExtraction:
)
finally:
set_turn_impersonated(False)
+ set_turn_trace_id(None)
assert create_mock.await_args is not None
extra_body = create_mock.await_args.kwargs["extra_body"]
@@ -1254,6 +1256,7 @@ class TestBaselineCostExtraction:
"generation_name": "copilot-baseline",
"environment": attribution_settings.config.app_env.value,
"impersonated": True,
+ "trace_id": "turn-1",
}
@pytest.mark.asyncio
diff --git a/autogpt_platform/backend/backend/copilot/executor/processor.py b/autogpt_platform/backend/backend/copilot/executor/processor.py
index 75676b9e7c..482e685cf8 100644
--- a/autogpt_platform/backend/backend/copilot/executor/processor.py
+++ b/autogpt_platform/backend/backend/copilot/executor/processor.py
@@ -32,7 +32,7 @@ from backend.copilot.expert_context import (
EXPERT_SESSION_TEMPORARY_MESSAGE,
ExpertSessionUnavailableError,
)
-from backend.copilot.llm_attribution import set_turn_impersonated
+from backend.copilot.llm_attribution import set_turn_impersonated, set_turn_trace_id
from backend.copilot.model import maybe_append_user_message
from backend.copilot.rate_limit import RateLimitUnavailable
from backend.copilot.response_model import (
@@ -837,6 +837,7 @@ class CoPilotProcessor:
envelope = taint_for_source_platform(entry.envelope, session)
set_turn_unattended(session, scheduled=entry.unattended)
set_turn_impersonated(entry.impersonated)
+ set_turn_trace_id(entry.turn_id)
set_turn_credential_pins(entry.credential_pins)
turn.engine_started(engine)
raw_stream = stream_fn(
diff --git a/autogpt_platform/backend/backend/copilot/executor/processor_test.py b/autogpt_platform/backend/backend/copilot/executor/processor_test.py
index 0f9fb0d1ed..5dd7017352 100644
--- a/autogpt_platform/backend/backend/copilot/executor/processor_test.py
+++ b/autogpt_platform/backend/backend/copilot/executor/processor_test.py
@@ -33,7 +33,7 @@ from backend.copilot.expert_context import (
EXPERT_SESSION_TEMPORARY_MESSAGE,
ExpertSessionUnavailableError,
)
-from backend.copilot.llm_attribution import is_turn_impersonated
+from backend.copilot.llm_attribution import is_turn_impersonated, turn_trace_id
from backend.copilot.model import ChatSession
from backend.copilot.rate_limit import UserPaywalledError
from backend.copilot.response_model import (
@@ -233,6 +233,7 @@ class TestExecuteAsyncScheduledTurnContext:
seen["unattended"] = is_unattended_turn()
seen["pins"] = dict(turn_credential_pins())
seen["impersonated"] = is_turn_impersonated()
+ seen["trace_id"] = turn_trace_id()
return MagicMock()
with (
@@ -275,13 +276,23 @@ class TestExecuteAsyncScheduledTurnContext:
seen = await self._seen_by_the_turn(entry)
- assert seen == {"unattended": True, "pins": {"exa": pin}, "impersonated": False}
+ assert seen == {
+ "unattended": True,
+ "pins": {"exa": pin},
+ "impersonated": False,
+ "trace_id": "turn-1",
+ }
@pytest.mark.asyncio
async def test_a_typed_turn_is_watched_and_has_no_pins(self) -> None:
seen = await self._seen_by_the_turn(_make_entry())
- assert seen == {"unattended": False, "pins": {}, "impersonated": False}
+ assert seen == {
+ "unattended": False,
+ "pins": {},
+ "impersonated": False,
+ "trace_id": "turn-1",
+ }
@pytest.mark.asyncio
async def test_an_impersonated_turn_is_marked_for_llm_analytics(self) -> None:
@@ -291,6 +302,15 @@ class TestExecuteAsyncScheduledTurnContext:
assert seen["impersonated"] is True
+ @pytest.mark.asyncio
+ async def test_the_turn_id_is_the_trace_of_its_llm_calls(self) -> None:
+ """Every model call of one reply groups into one PostHog trace."""
+ entry = _make_entry().model_copy(update={"turn_id": "turn-42"})
+
+ seen = await self._seen_by_the_turn(entry)
+
+ assert seen["trace_id"] == "turn-42"
+
class TestExecuteAsyncAclose:
"""``_execute_async`` must call ``aclose`` on the published stream both
diff --git a/autogpt_platform/backend/backend/copilot/feedback.py b/autogpt_platform/backend/backend/copilot/feedback.py
index 10400ccd27..e088473864 100644
--- a/autogpt_platform/backend/backend/copilot/feedback.py
+++ b/autogpt_platform/backend/backend/copilot/feedback.py
@@ -20,9 +20,10 @@ consent, like the generations OpenRouter Broadcast sends there.
``$ai_session_id`` is the chat session id, the same value the chat's
``$ai_generation`` events carry, which is what joins a rating to the
generations of its chat; ``message_id`` names the reply. It carries no
-``$ai_trace_id``: Broadcast assigns the generations' trace ids, so we have
-none to match. Rating again sends another event; the latest per
-``feedback_id`` is the rating that stands.
+``$ai_trace_id`` yet: each generation's trace id is the turn that made it (see
+``llm_attribution``), but a reply row does not record which turn wrote it.
+Rating again sends another event; the latest per ``feedback_id`` is the
+rating that stands.
"""
import logging
diff --git a/autogpt_platform/backend/backend/copilot/llm_attribution.py b/autogpt_platform/backend/backend/copilot/llm_attribution.py
index 1f555493e8..8503ecf6a7 100644
--- a/autogpt_platform/backend/backend/copilot/llm_attribution.py
+++ b/autogpt_platform/backend/backend/copilot/llm_attribution.py
@@ -8,9 +8,15 @@ OpenRouter Broadcast sends every generation to PostHog LLM analytics as an
the distinct id every other backend and browser event uses, so the
generation lands on the same person as their product events and replays.
- ``session_id`` becomes ``$ai_session_id``: the chat session id.
+- ``trace.trace_id`` becomes ``$ai_trace_id``: the id of the turn that made
+ the call, so every model call behind one AutoPilot reply (the main loop,
+ sub-agents, compaction, the chat title) groups into one trace.
- ``trace.generation_name`` becomes ``$ai_span_name``; every other ``trace``
key arrives as ``metadata_<key>`` (``metadata_environment``,
- ``metadata_impersonated``).
+ ``metadata_impersonated``, and ``metadata_revision``: the deployed build's
+ git sha from ``config.sentry_release``, left out when that is blank).
+ Broadcast sends these events itself, not through our PostHog client, so
+ they never get a property the client adds to every event it captures.
Only calls that go through OpenRouter are covered. Direct Anthropic, Claude
Code subscription and user-connected (Codex) routes never pass OpenRouter, so
@@ -37,6 +43,11 @@ GenerationName = Literal["copilot-title", "copilot-sdk", "copilot-baseline"]
# executor sets it for every turn, so a value never carries over.
_turn_impersonated: ContextVar[bool] = ContextVar("_turn_impersonated", default=False)
+# The running turn's id, sent as the trace id of every call it makes. The
+# executor sets it for every turn, so a value never carries over; a call made
+# outside a turn sends none.
+_turn_trace_id: ContextVar[str | None] = ContextVar("_turn_trace_id", default=None)
+
def openrouter_attribution(
*,
@@ -52,6 +63,12 @@ def openrouter_attribution(
"impersonated": is_turn_impersonated(),
}
}
+ revision = settings.config.sentry_release.strip()
+ if revision:
+ body["trace"]["revision"] = revision
+ trace_id = turn_trace_id()
+ if trace_id:
+ body["trace"]["trace_id"] = trace_id[:_MAX_ID_CHARS]
if user_id:
body["user"] = user_id[:_MAX_ID_CHARS]
if session_id:
@@ -68,6 +85,15 @@ def is_turn_impersonated() -> bool:
return _turn_impersonated.get()
+def set_turn_trace_id(turn_id: str | None) -> None:
+ """Make *turn_id* the trace of every model call the running turn makes."""
+ _turn_trace_id.set(turn_id or None)
+
+
+def turn_trace_id() -> str | None:
+ return _turn_trace_id.get()
+
+
def is_impersonated_request(request: Request) -> bool:
"""Whether an admin made *request* as another user.
diff --git a/autogpt_platform/backend/backend/copilot/llm_attribution_test.py b/autogpt_platform/backend/backend/copilot/llm_attribution_test.py
index 02964b6cd6..0ba9d68a8d 100644
--- a/autogpt_platform/backend/backend/copilot/llm_attribution_test.py
+++ b/autogpt_platform/backend/backend/copilot/llm_attribution_test.py
@@ -11,15 +11,19 @@ from backend.copilot.llm_attribution import (
is_turn_impersonated,
openrouter_attribution,
set_turn_impersonated,
+ set_turn_trace_id,
+ turn_trace_id,
)
from backend.util.settings import AppEnvironment
@pytest.fixture(autouse=True)
-def _not_impersonated():
+def _outside_a_turn():
set_turn_impersonated(False)
+ set_turn_trace_id(None)
yield
set_turn_impersonated(False)
+ set_turn_trace_id(None)
def test_attribution_names_the_user_chat_and_environment(mocker):
@@ -51,6 +55,25 @@ def test_attribution_without_a_user_or_chat_still_tags_the_environment():
assert body["trace"]["generation_name"] == "copilot-baseline"
+ ("sentry_release", "expected"),
+ [("0123abcd\n", {"revision": "0123abcd"}), ("", {})],
+ ids=["revision set", "no revision"],
+)
+def test_attribution_names_the_revision_that_made_the_call(
+ mocker, sentry_release: str, expected: dict[str, str]
+):
+ mocker.patch.object(
+ llm_attribution.settings.config, "sentry_release", sentry_release
+ )
+
+ body = openrouter_attribution(
+ user_id=None, session_id=None, generation_name="copilot-sdk"
+ )
+
+ assert {k: v for k, v in body["trace"].items() if k == "revision"} == expected
+
+
def test_attribution_marks_an_impersonated_turn():
set_turn_impersonated(True)
@@ -75,6 +98,62 @@ async def test_impersonation_does_not_leak_into_another_turn():
assert (admin, user) == (True, False)
+def test_attribution_groups_a_turns_calls_under_its_trace():
+ set_turn_trace_id("turn-1")
+
+ body = openrouter_attribution(
+ user_id="user-1", session_id="sess-1", generation_name="copilot-sdk"
+ )
+
+ assert body["trace"]["trace_id"] == "turn-1"
+
+
+def test_attribution_outside_a_turn_sends_no_trace_id():
+ body = openrouter_attribution(
+ user_id="user-1", session_id="sess-1", generation_name="copilot-title"
+ )
+
+ assert "trace_id" not in body["trace"]
+
+
+def test_a_blank_turn_id_sends_no_trace_id():
+ set_turn_trace_id("")
+
+ assert turn_trace_id() is None
+ body = openrouter_attribution(
+ user_id="user-1", session_id="sess-1", generation_name="copilot-sdk"
+ )
+ assert "trace_id" not in body["trace"]
+
+
+async def test_a_trace_id_does_not_leak_into_another_turn():
+ async def turn(turn_id: str) -> str | None:
+ set_turn_trace_id(turn_id)
+ await asyncio.sleep(0)
+ return openrouter_attribution(
+ user_id="user-1", session_id="sess-1", generation_name="copilot-sdk"
+ )["trace"]["trace_id"]
+
+ first, second = await asyncio.gather(
+ asyncio.create_task(turn("turn-1")), asyncio.create_task(turn("turn-2"))
+ )
+
+ assert (first, second) == ("turn-1", "turn-2")
+
+
+async def test_a_task_the_turn_starts_shares_its_trace():
+ """The chat title is generated in a task the first turn starts, so its
+ call lands in that turn's trace."""
+ set_turn_trace_id("turn-1")
+
+ async def title_call() -> str | None:
+ return turn_trace_id()
+
+ assert await asyncio.create_task(title_call()) == "turn-1"
+
+
@pytest.mark.parametrize(
("headers", "expected"),
[
diff --git a/autogpt_platform/backend/backend/copilot/sdk/env_broadcast_test.py b/autogpt_platform/backend/backend/copilot/sdk/env_broadcast_test.py
index 7094f4cf6a..fa3ad7dbd5 100644
--- a/autogpt_platform/backend/backend/copilot/sdk/env_broadcast_test.py
+++ b/autogpt_platform/backend/backend/copilot/sdk/env_broadcast_test.py
@@ -11,7 +11,7 @@ from unittest.mock import patch
import pytest
from backend.copilot.config import ChatConfig
-from backend.copilot.llm_attribution import set_turn_impersonated
+from backend.copilot.llm_attribution import set_turn_impersonated, set_turn_trace_id
from backend.copilot.llm_attribution import settings as attribution_settings
from backend.copilot.sdk.env import build_sdk_env
@@ -31,10 +31,12 @@ def _config(**overrides) -> ChatConfig:
@pytest.fixture(autouse=True)
-def _not_impersonated():
+def _outside_a_turn():
set_turn_impersonated(False)
+ set_turn_trace_id(None)
yield
set_turn_impersonated(False)
+ set_turn_trace_id(None)
def _extra_body(**kwargs) -> dict:
@@ -73,6 +75,16 @@ def test_an_impersonated_turn_is_marked():
assert body["trace"]["impersonated"] is True
+def test_every_request_of_a_turn_carries_its_trace_id():
+ """Broadcast maps ``trace.trace_id`` to ``$ai_trace_id``, so the turn's
+ main, sub-agent and compaction calls group into one PostHog trace."""
+ set_turn_trace_id("turn-1")
+
+ body = _extra_body(session_id="sess-1", user_id="user-1")
+
+ assert body["trace"]["trace_id"] == "turn-1"
+
+
def test_no_user_sends_no_user_fields():
body = _extra_body(session_id="sess-1")
diff --git a/autogpt_platform/backend/backend/copilot/service_unit_test.py b/autogpt_platform/backend/backend/copilot/service_unit_test.py
index e9b66293ef..4593b1cdfc 100644
--- a/autogpt_platform/backend/backend/copilot/service_unit_test.py
+++ b/autogpt_platform/backend/backend/copilot/service_unit_test.py
@@ -21,6 +21,7 @@ from openai.types.chat.chat_completion import Choice
from openai.types.chat.chat_completion_message import ChatCompletionMessage
from openai.types.completion_usage import CompletionUsage
+from backend.copilot.llm_attribution import set_turn_trace_id
from backend.copilot.service import (
_TITLE_TIMEOUT_SECONDS,
_fallback_title_from_message,
@@ -674,6 +675,35 @@ class TestGenerateSessionTitle:
"impersonated": False,
}
+ @pytest.mark.asyncio
+ async def test_title_call_lands_in_the_trace_of_the_turn_that_asked(self):
+ """The first turn generates the title, so the title call groups with
+ that reply's other calls in PostHog."""
+ resp = _build_completion(content="Title")
+ helper = AsyncMock(return_value=resp)
+ with (
+ patch(
+ "backend.copilot.service.call_provider_openai_compat_sync",
+ new=helper,
+ ),
+ patch(
+ "backend.copilot.service.config",
+ MagicMock(
+ aux_uses_openrouter=True,
+ title_model="anthropic/claude-haiku",
+ ),
+ ),
+ ):
+ set_turn_trace_id("turn-1")
+ try:
+ await _generate_session_title(
+ "hello world", user_id="user-abc", session_id="sess-abc"
+ )
+ finally:
+ set_turn_trace_id(None)
+ extra_body = helper.await_args.kwargs["extra_body"]
+ assert extra_body["trace"]["trace_id"] == "turn-1"
+
@pytest.mark.asyncio
async def test_title_call_off_openrouter_carries_no_broadcast_fields(self):
"""Broadcast only sees OpenRouter calls, and a direct-Anthropic aux
diff --git a/docs/engineering/tracking-plan.md b/docs/engineering/tracking-plan.md
index c0e4112eac..db992cee10 100644
--- a/docs/engineering/tracking-plan.md
+++ b/docs/engineering/tracking-plan.md
@@ -315,8 +315,8 @@ PostHog's own, not the analytics plan's.
| Event | Sender | Status | Required properties | Fires when |
| --- | --- | --- | --- | --- |
-| `$ai_generation` | OpenRouter Broadcast | live | set by OpenRouter from the request (`backend/copilot/llm_attribution.py`): `distinct_id` = the request's `user` (the platform user id), `$ai_session_id` = its `session_id` (the chat session id), `$ai_span_name` = `trace.generation_name` (`copilot-title`, `copilot-sdk`, `copilot-baseline`), `metadata_environment` (`local`, `dev`, `prod`), `metadata_impersonated`; model, tokens, cost and latency from OpenRouter | An AutoPilot LLM call goes through OpenRouter: a session title, an SDK turn (main agent, sub-agents and the CLI's compaction, via `CLAUDE_CODE_EXTRA_BODY`) or a baseline turn. |
-| `$ai_metric` | backend | live | `$ai_metric_name` (`user-feedback`, `copy`), `$ai_metric_value` (`1` thumbs up or copy, `0` thumbs down), `$ai_session_id` and `chat_session_id` (the chat session id), `message_id` (the reply's row id), `message_sequence`, `feedback_id` (the stored rating, shared with its Langfuse score), `impersonated`; `$ai_feedback_text` when a thumbs-down has a comment | A reply is rated (`POST /api/chat/sessions/{id}/feedback`, `copilot/feedback.py`). Rating a reply again sends another event; the latest per `feedback_id` stands. |
+| `$ai_generation` | OpenRouter Broadcast | live | set by OpenRouter from the request (`backend/copilot/llm_attribution.py`): `distinct_id` = the request's `user` (the platform user id), `$ai_session_id` = its `session_id` (the chat session id), `$ai_span_name` = `trace.generation_name` (`copilot-title`, `copilot-sdk`, `copilot-baseline`), `$ai_trace_id` = `trace.trace_id` (the id of the executor turn that made the call, so all calls behind one reply share it; absent for a call made outside a turn), `metadata_environment` (`local`, `dev`, `prod`), `metadata_impersonated`; `metadata_revision` (the deployed build's git sha, `config.sentry_release`) when it is set; model, tokens, cost and latency from OpenRouter | An AutoPilot LLM call goes through OpenRouter: a session title, an SDK turn (main agent, sub-agents and the CLI's compaction, via `CLAUDE_CODE_EXTRA_BODY`) or a baseline turn. |
+| `$ai_metric` | backend | live | `$ai_metric_name` (`user-feedback`, `copy`), `$ai_metric_value` (`1` thumbs up or copy, `0` thumbs down), `$ai_session_id` and `chat_session_id` (the chat session id), `message_id` (the reply's row id), `message_sequence`, `feedback_id` (the stored rating, shared with its Langfuse score), `impersonated`; `$ai_feedback_text` when a thumbs-down has a comment. No `$ai_trace_id` (see below) | A reply is rated (`POST /api/chat/sessions/{id}/feedback`, `copilot/feedback.py`). Rating a reply again sends another event; the latest per `feedback_id` stands. |
What joins them:
@@ -325,11 +325,20 @@ What joins them:
both land on the person with their product events and replays.
- **Chat.** `$ai_session_id` is the chat session id on both, so a chat's
generations and its ratings group in the Sessions tab and join on it.
- `message_id` says which reply was rated; a rating carries no
- `$ai_trace_id`, since Broadcast assigns the generations' trace ids.
+ `message_id` says which reply was rated. A rating carries no
+ `$ai_trace_id` yet: a generation's trace id is the turn that made it, but a
+ reply row does not record which turn wrote it, so a rating joins its
+ reply's generations only through the chat.
+- **Reply.** All generations behind one reply (the main loop, sub-agents,
+ compaction and, on a chat's first reply, the title) share `$ai_trace_id`,
+ the executor turn id, so PostHog groups them into one trace. An expert
+ subtask runs as a turn of its own and gets its own trace.
- **Environment.** OpenRouter forwards our `trace` keys as `metadata_<key>`,
so a generation's environment is `metadata_environment`, not the
`environment` base property; `$ai_metric` has both base properties as usual.
+ Likewise the build is `metadata_revision` on a generation; `$ai_metric`
+ sets no build property of its own and carries only the base properties our
+ PostHog client adds to every event.
- **Impersonation.** A turn or rating an admin made while impersonating the
user (`X-Act-As-User-Id`) is still sent, as the user, with
`metadata_impersonated` / `impersonated` true, so it can be filtered out.