AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
feat(backend/copilot): attribute AutoPilot generations in PostHog and send reply feedback there (#15498)
Krzysztof Czerwinski40d83480a221
diff --git a/autogpt_platform/backend/backend/api/features/chat/feedback.py b/autogpt_platform/backend/backend/api/features/chat/feedback.pyindex d04f06f8a2..7bea52fead 100644--- a/autogpt_platform/backend/backend/api/features/chat/feedback.py+++ b/autogpt_platform/backend/backend/api/features/chat/feedback.py@@ -7,7 +7,7 @@ logic lives in ``backend.copilot.feedback``.from typing import Annotatedfrom autogpt_libs import auth-from fastapi import APIRouter, HTTPException, Security+from fastapi import APIRouter, HTTPException, Request, Securityfrom pydantic import BaseModel, Field, field_validator, model_validatorfrom backend.copilot.feedback import (@@ -17,6 +17,7 @@ from backend.copilot.feedback import (MessageFeedbackResponse,record_message_feedback,)+from backend.copilot.llm_attribution import is_impersonated_requestrouter = APIRouter(tags=["chat"])@@ -71,14 +72,16 @@ async def submit_message_feedback(session_id: str,request: MessageFeedbackRequest,user_id: Annotated[str, Security(auth.get_user_id)],+ http_request: Request,) -> MessageFeedbackResponse:"""Rate a reply in one of the caller's chats.Saves the rating, one per reply and score name so rating againoverwrites it, and scores it in Langfuse on the trace of the turn thatwrote the reply, or on the chat's Langfuse session when that trace is- not known. Langfuse being down does not fail the request; the response- says where the score went.+ not known. It also goes to PostHog LLM analytics as an ``$ai_metric``+ event. Langfuse or PostHog being down does not fail the request; the+ response says where the Langfuse score went."""feedback = await record_message_feedback(user_id=user_id,@@ -87,6 +90,7 @@ async def submit_message_feedback(score_name=request.score_name,score_value=request.score_value,comment=request.comment,+ impersonated=is_impersonated_request(http_request),)if feedback is None:raise HTTPException(diff --git a/autogpt_platform/backend/backend/api/features/chat/feedback_test.py b/autogpt_platform/backend/backend/api/features/chat/feedback_test.pyindex fc88c67cc8..608e4aa3c2 100644--- a/autogpt_platform/backend/backend/api/features/chat/feedback_test.py+++ b/autogpt_platform/backend/backend/api/features/chat/feedback_test.py@@ -12,6 +12,7 @@ import fastapiimport fastapi.testclientimport pytestimport pytest_mock+from autogpt_libs.auth.jwt_utils import get_jwt_payloadfrom langfuse.api import ScoreDataTypefrom prisma.models import ChatMessage as PrismaChatMessage@@ -31,8 +32,6 @@ FEEDBACK_ID = "c3a1b2d4-e5f6-4789-8abc-def012345678"@pytest.fixture(autouse=True)def setup_app_auth(mock_jwt_user):- from autogpt_libs.auth.jwt_utils import get_jwt_payload-app.dependency_overrides[get_jwt_payload] = mock_jwt_user["get_jwt_payload"]yieldapp.dependency_overrides.clear()@@ -46,6 +45,13 @@ def langfuse_settings(mocker: pytest_mock.MockerFixture):mocker.patch.object(secrets, "langfuse_tracing_environment", "dev")[email protected](autouse=True)+def posthog_capture(mocker: pytest_mock.MockerFixture) -> MagicMock:+ """PostHog is covered in ``backend/copilot/feedback_test.py``; here it+ only must not reach a real client."""+ return mocker.patch.object(feedback_service, "capture")++@pytest.fixturedef score_create(mocker: pytest_mock.MockerFixture) -> AsyncMock:"""The Langfuse score API call, with the client it hangs off mocked."""@@ -355,6 +361,48 @@ def test_invalid_ratings_are_rejected(upsert.assert_not_awaited()+def test_rating_is_sent_to_posthog_as_the_user(+ mocker: pytest_mock.MockerFixture,+ test_user_id: str,+ upsert: AsyncMock,+ score_create: AsyncMock,+ posthog_capture: MagicMock,+) -> None:+ _mock_message(mocker, _reply(trace_id=TRACE_ID))++ response = _post(_thumbs_down())++ assert response.status_code == 200+ distinct_id, event, properties = posthog_capture.call_args.args+ assert distinct_id == test_user_id+ assert event == "$ai_metric"+ assert properties["impersonated"] is False+ score_create.assert_awaited_once()+++def test_rating_by_an_impersonating_admin_is_marked(+ mocker: pytest_mock.MockerFixture,+ mock_jwt_admin,+ target_user_id: str,+ upsert: AsyncMock,+ score_create: AsyncMock,+ posthog_capture: MagicMock,+) -> None:+ app.dependency_overrides[get_jwt_payload] = mock_jwt_admin["get_jwt_payload"]+ _mock_message(mocker, _reply(trace_id=TRACE_ID))++ response = client.post(+ f"/sessions/{SESSION_ID}/feedback",+ json=_thumbs_down(),+ headers={"X-Act-As-User-Id": target_user_id},+ )++ assert response.status_code == 200+ distinct_id, _, properties = posthog_capture.call_args.args+ assert distinct_id == target_user_id+ assert properties["impersonated"] is True++def test_blank_comment_is_saved_as_none(mocker: pytest_mock.MockerFixture,upsert: AsyncMock,diff --git a/autogpt_platform/backend/backend/api/features/chat/routes.py b/autogpt_platform/backend/backend/api/features/chat/routes.pyindex 209d87a987..8563374685 100644--- a/autogpt_platform/backend/backend/api/features/chat/routes.py+++ b/autogpt_platform/backend/backend/api/features/chat/routes.py@@ -8,7 +8,7 @@ from typing import Annotated, Any, Literalfrom uuid import uuid4from autogpt_libs import auth-from fastapi import APIRouter, HTTPException, Query, Response, Security+from fastapi import APIRouter, HTTPException, Query, Request, Response, Securityfrom fastapi.responses import JSONResponse, StreamingResponsefrom pydantic import BaseModel, ConfigDict, Field, field_validator@@ -41,6 +41,7 @@ from backend.copilot.expert_kickoff import (expert_kickoff_metadata,scoped_client_message_id,)+from backend.copilot.llm_attribution import is_impersonated_requestfrom backend.copilot.model import (CHAT_STATUS_IDLE,CHAT_STATUS_RUNNING,@@ -1750,6 +1751,8 @@ async def stream_chat_post(request: StreamChatRequest,user_id: str = Security(auth.get_user_id),ctx: auth.RequestContext = Security(auth.get_request_context),+ *,+ http_request: Request,):"""Start a new turn and return an AI SDK UI message stream.@@ -2044,6 +2047,7 @@ async def stream_chat_post(llm_credential_id=session.metadata.llm_credential_id,permissions=builder_permissions,request_arrival_at=request_arrival_at,+ impersonated=is_impersonated_request(http_request),)except ConcurrentTurnLimitError as exc:if resume_persisted_kickoff:@@ -2074,6 +2078,7 @@ async def stream_chat_post(else None),request_arrival_at=request_arrival_at,+ impersonated=is_impersonated_request(http_request),)except turn_queue.InflightCapExceeded:await release_client_message_claim()diff --git a/autogpt_platform/backend/backend/api/features/chat/routes_test.py b/autogpt_platform/backend/backend/api/features/chat/routes_test.pyindex 4ab3d61c54..7f7cab322f 100644--- a/autogpt_platform/backend/backend/api/features/chat/routes_test.py+++ b/autogpt_platform/backend/backend/api/features/chat/routes_test.py@@ -11,6 +11,7 @@ import fastapi.testclientimport httpximport pytestimport pytest_mock+from autogpt_libs.auth.jwt_utils import get_jwt_payloadfrom pydantic import SecretStrfrom sentry_sdk.integrations.asgi import SentryAsgiMiddleware@@ -73,7 +74,6 @@ def test_tool_response_union_exports_expert_soul_response() -> None:def setup_app_auth(mock_jwt_user, mocker: pytest_mock.MockerFixture):"""Setup auth overrides for all tests in this module"""from autogpt_libs.auth.dependencies import get_request_context- from autogpt_libs.auth.jwt_utils import get_jwt_payloadapp.dependency_overrides[get_jwt_payload] = mock_jwt_user["get_jwt_payload"]app.dependency_overrides[get_request_context] = mock_jwt_user["get_request_context"]@@ -431,6 +431,78 @@ def test_stream_chat_accepts_20_file_ids(mocker: pytest_mock.MockerFixture):assert response.status_code == 200+def _mock_rate_limits_for_an_unseeded_user(mocker: pytest_mock.MockerFixture) -> None:+ """The impersonated user has no row in the test database, so the tier+ lookup behind the rate limit would fail before the turn is scheduled."""+ mocker.patch(+ "backend.api.features.chat.routes.get_global_rate_limits",+ new_callable=AsyncMock,+ return_value=(0, 0, SubscriptionTier.BASIC),+ )+ mocker.patch(+ "backend.api.features.chat.routes.check_rate_limit", new_callable=AsyncMock+ )+++def test_stream_chat_turn_is_not_impersonated_by_default(+ mocker: pytest_mock.MockerFixture,+) -> None:+ mocks = _mock_stream_internals(mocker)++ response = client.post("/sessions/sess-1/stream", json={"message": "hello"})++ assert response.status_code == 200+ assert mocks.enqueue.await_args.kwargs["impersonated"] is False+++def test_stream_chat_marks_a_turn_an_impersonating_admin_starts(+ mocker: pytest_mock.MockerFixture,+ mock_jwt_admin,+ target_user_id: str,+) -> None:+ """The turn still runs and is still tracked, as the user, but its LLM+ analytics say an admin sent it."""+ app.dependency_overrides[get_jwt_payload] = mock_jwt_admin["get_jwt_payload"]+ mocks = _mock_stream_internals(mocker)+ _mock_rate_limits_for_an_unseeded_user(mocker)++ response = client.post(+ "/sessions/sess-1/stream",+ json={"message": "hello"},+ headers={"X-Act-As-User-Id": target_user_id},+ )++ assert response.status_code == 200+ kwargs = mocks.enqueue.await_args.kwargs+ assert kwargs["user_id"] == target_user_id+ assert kwargs["impersonated"] is True+++def test_stream_chat_queues_an_impersonated_turn_as_impersonated(+ mocker: pytest_mock.MockerFixture,+ mock_jwt_admin,+ target_user_id: str,+) -> None:+ """A turn queued behind the running cap is promoted later, from the stored+ row, so the row has to say an admin sent it."""+ app.dependency_overrides[get_jwt_payload] = mock_jwt_admin["get_jwt_payload"]+ mocks = _mock_stream_internals(mocker)+ _mock_rate_limits_for_an_unseeded_user(mocker)+ mocks.enqueue.side_effect = chat_routes.ConcurrentTurnLimitError("busy")+ queue = mocker.patch.object(+ chat_routes.turn_queue, "try_enqueue_turn", new_callable=AsyncMock+ )++ client.post(+ "/sessions/sess-1/stream",+ json={"message": "hello"},+ headers={"X-Act-As-User-Id": target_user_id},+ )++ assert queue.await_args.kwargs["user_id"] == target_user_id+ assert queue.await_args.kwargs["impersonated"] is True++def test_stream_chat_allows_an_active_expert_session(mocker: pytest_mock.MockerFixture,test_user_id: str,diff --git a/autogpt_platform/backend/backend/copilot/baseline/service.py b/autogpt_platform/backend/backend/copilot/baseline/service.pyindex fe44f3ea9e..d44719a3a9 100644--- a/autogpt_platform/backend/backend/copilot/baseline/service.py+++ b/autogpt_platform/backend/backend/copilot/baseline/service.py@@ -55,6 +55,7 @@ from backend.copilot.gate.held import resolve_answeredfrom backend.copilot.graphiti.config import is_enabled_for_userfrom backend.copilot.graphiti.context import fetch_warm_contextfrom backend.copilot.graphiti.ingest import enqueue_conversation_turn+from backend.copilot.llm_attribution import openrouter_attributionfrom backend.copilot.local_context_probe import (compaction_target_for_window,probe_local_context_window,@@ -510,6 +511,10 @@ class _BaselineStreamState:# what already ran; see backend/copilot/segments.py.llm_auth_provider: CopilotLlmAuthProvider | None = Nonellm_credential_id: str | None = None+ # Whose turn this is and in which chat, sent on OpenRouter requests so+ # Broadcast attributes each generation (see ``llm_attribution``).+ user_id: str | None = None+ session_id: str | None = None# Live delivery channel drained concurrently by ``stream_chat_completion_baseline``# so reasoning / text / tool events reach the SSE wire **during** the upstream# LLM stream, not after ``_baseline_llm_caller`` returns. Before this was a@@ -916,8 +921,16 @@ async def _baseline_llm_caller(# which stricter local backends reject outright.passelif is_openrouter_transport:- # OR-only: Anthropic's compat endpoint 400s on usage.include + reasoning.+ # OR-only: Anthropic's compat endpoint 400s on usage.include + reasoning+ # and on the Broadcast fields, which only OpenRouter reads anyway.extra_body.update(dict(_OPENROUTER_INCLUDE_USAGE_COST))+ extra_body.update(+ openrouter_attribution(+ user_id=state.user_id,+ session_id=state.session_id,+ generation_name="copilot-baseline",+ )+ )reasoning_param = reasoning_extra_body(state.model, config.claude_agent_max_thinking_tokens)@@ -2393,6 +2406,8 @@ async def stream_chat_completion_baseline(llm_credential_id=(session.metadata.llm_credential_id if session is not None else None),+ user_id=user_id,+ session_id=session_id,)# Queued, not yielded: the loop below yields them inside the try whosediff --git a/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py b/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.pyindex eaac3e8348..b509950cb1 100644--- a/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py+++ b/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py@@ -37,6 +37,8 @@ from backend.copilot.baseline.service import ()from backend.copilot.context import get_execution_context, set_execution_contextfrom backend.copilot.expert_context import ExpertSessionUnavailableError+from backend.copilot.llm_attribution import set_turn_impersonated+from backend.copilot.llm_attribution import settings as attribution_settingsfrom backend.copilot.model import ChatMessage, ChatSessionfrom backend.copilot.model_router import ResolvedModelfrom backend.copilot.response_model import (@@ -1208,9 +1210,80 @@ class TestBaselineCostExtraction:create_mock.assert_awaited_once()await_args = create_mock.await_argsassert await_args is not None- assert await_args.kwargs["extra_body"] == {"usage": {"include": True}}+ assert await_args.kwargs["extra_body"]["usage"] == {"include": True}assert await_args.kwargs["stream_options"] == {"include_usage": True}+ @pytest.mark.asyncio+ async def test_openrouter_request_attributes_the_turn_for_broadcast(self):+ """OpenRouter Broadcast puts the generation on the user's PostHog+ person and the chat's ``$ai_session_id`` from these fields."""+ state = _BaselineStreamState(+ model="gpt-4o-mini", user_id="user-1", session_id="sess-1"+ )+ create_mock = AsyncMock(return_value=_make_stream_mock())+ mock_client = MagicMock()+ mock_client.chat.completions.create = create_mock++ with (+ patch(+ "backend.copilot.baseline.service._get_main_client",+ return_value=mock_client,+ ),+ patch("backend.copilot.baseline.service.config.use_openrouter", True),+ patch("backend.copilot.baseline.service.config.api_key", "or-key"),+ patch(+ "backend.copilot.baseline.service.config.base_url",+ "https://openrouter.ai/api/v1",+ ),+ ):+ set_turn_impersonated(True)+ try:+ await _baseline_llm_caller(+ messages=[{"role": "user", "content": "hi"}],+ tools=[],+ state=state,+ )+ finally:+ set_turn_impersonated(False)++ assert create_mock.await_args is not None+ extra_body = create_mock.await_args.kwargs["extra_body"]+ assert extra_body["user"] == "user-1"+ assert extra_body["session_id"] == "sess-1"+ assert extra_body["trace"] == {+ "generation_name": "copilot-baseline",+ "environment": attribution_settings.config.app_env.value,+ "impersonated": True,+ }++ @pytest.mark.asyncio+ async def test_direct_anthropic_request_carries_no_broadcast_fields(self):+ """Anthropic's OpenAI-compat endpoint 400s on unknown body fields,+ and Broadcast never sees a call that skips OpenRouter."""+ state = _BaselineStreamState(+ model="claude-sonnet-4-6", user_id="user-1", session_id="sess-1"+ )+ create_mock = AsyncMock(return_value=_make_stream_mock())+ mock_client = MagicMock()+ mock_client.chat.completions.create = create_mock++ with (+ patch(+ "backend.copilot.baseline.service._get_main_client",+ return_value=mock_client,+ ),+ patch("backend.copilot.baseline.service.config.use_openrouter", False),+ ):+ await _baseline_llm_caller(+ messages=[{"role": "user", "content": "hi"}],+ tools=[],+ state=state,+ )++ assert create_mock.await_args is not None+ extra_body = create_mock.await_args.kwargs["extra_body"] or {}+ assert not {"user", "session_id", "trace"} & extra_body.keys()+class TestMidLoopPendingFlushOrdering:"""Regression test for the mid-loop pending drain ordering invariant.diff --git a/autogpt_platform/backend/backend/copilot/executor/processor.py b/autogpt_platform/backend/backend/copilot/executor/processor.pyindex 1d01cd47ac..3ce2274e77 100644--- a/autogpt_platform/backend/backend/copilot/executor/processor.py+++ b/autogpt_platform/backend/backend/copilot/executor/processor.py@@ -25,6 +25,7 @@ from backend.copilot.expert_context import (EXPERT_SESSION_TEMPORARY_MESSAGE,ExpertSessionUnavailableError,)+from backend.copilot.llm_attribution import set_turn_impersonatedfrom backend.copilot.response_model import StreamError, StreamStatusfrom backend.copilot.sdk import service as sdk_servicefrom backend.copilot.sdk.dummy import stream_chat_completion_dummy@@ -742,6 +743,7 @@ class CoPilotProcessor:# same events as they are produced.envelope = taint_for_source_platform(entry.envelope, session)set_turn_unattended(session, scheduled=entry.unattended)+ set_turn_impersonated(entry.impersonated)set_turn_credential_pins(entry.credential_pins)turn.engine_started(engine)raw_stream = stream_fn(diff --git a/autogpt_platform/backend/backend/copilot/executor/processor_test.py b/autogpt_platform/backend/backend/copilot/executor/processor_test.pyindex dd2610123a..ff82c5a98d 100644--- a/autogpt_platform/backend/backend/copilot/executor/processor_test.py+++ b/autogpt_platform/backend/backend/copilot/executor/processor_test.py@@ -33,6 +33,7 @@ from backend.copilot.expert_context import (EXPERT_SESSION_TEMPORARY_MESSAGE,ExpertSessionUnavailableError,)+from backend.copilot.llm_attribution import is_turn_impersonatedfrom backend.copilot.model import ChatSessionfrom backend.copilot.rate_limit import UserPaywalledErrorfrom backend.copilot.response_model import (@@ -218,6 +219,7 @@ class TestExecuteAsyncScheduledTurnContext:def stream(**_kwargs):seen["unattended"] = is_unattended_turn()seen["pins"] = dict(turn_credential_pins())+ seen["impersonated"] = is_turn_impersonated()return MagicMock()with (@@ -260,13 +262,21 @@ class TestExecuteAsyncScheduledTurnContext:seen = await self._seen_by_the_turn(entry)- assert seen == {"unattended": True, "pins": {"exa": pin}}+ assert seen == {"unattended": True, "pins": {"exa": pin}, "impersonated": False}@pytest.mark.asyncioasync def test_a_typed_turn_is_watched_and_has_no_pins(self) -> None:seen = await self._seen_by_the_turn(_make_entry())- assert seen == {"unattended": False, "pins": {}}+ assert seen == {"unattended": False, "pins": {}, "impersonated": False}++ @pytest.mark.asyncio+ async def test_an_impersonated_turn_is_marked_for_llm_analytics(self) -> None:+ entry = _make_entry().model_copy(update={"impersonated": True})++ seen = await self._seen_by_the_turn(entry)++ assert seen["impersonated"] is Trueclass TestExecuteAsyncAclose:diff --git a/autogpt_platform/backend/backend/copilot/executor/utils.py b/autogpt_platform/backend/backend/copilot/executor/utils.pyindex 4cb63f67d2..b17beb0307 100644--- a/autogpt_platform/backend/backend/copilot/executor/utils.py+++ b/autogpt_platform/backend/backend/copilot/executor/utils.py@@ -336,6 +336,11 @@ class CoPilotExecutionEntry(BaseModel):is the user's own. Tools then never hand a question back to the user(SECRT-2804); see ``set_turn_unattended``."""+ impersonated: bool = False+ """An admin impersonating the user sent the message that started this+ turn. Marks the turn's LLM analytics rather than dropping them; see+ ``llm_attribution.set_turn_impersonated``."""+credential_pins: CredentialPins = Field(default_factory=dict)"""``{provider: pin}``: the accounts the schedule that fired this turn wasset up to run on. They win over the chat's own picks, and a pinned account@@ -393,6 +398,7 @@ async def enqueue_copilot_turn(*,envelope: TurnEnvelope,unattended: bool = False,+ impersonated: bool = False,credential_pins: CredentialPins | None = None,scheduled: ScheduledTurnOrigin | None = None,trace_headers: Mapping[str, str] | None = None,@@ -443,6 +449,7 @@ async def enqueue_copilot_turn(message_metadata=message_metadata,envelope=envelope,unattended=unattended,+ impersonated=impersonated,credential_pins=credential_pins or {},scheduled=scheduled,trace_headers=(@@ -485,6 +492,7 @@ async def schedule_turn(credential_pins: CredentialPins | None = None,scheduled: ScheduledTurnOrigin | None = None,delegated: bool = False,+ impersonated: bool = False,) -> None:"""End-to-end "start a copilot turn": reserve a per-user concurrencyslot, register the session in the stream registry, then publish the@@ -556,6 +564,7 @@ async def schedule_turn(unattended=unattended,credential_pins=credential_pins,scheduled=scheduled,+ impersonated=impersonated,)@@ -585,6 +594,7 @@ async def dispatch_turn(scheduled: ScheduledTurnOrigin | None = None,root: bool = False,envelope: TurnEnvelope | None = None,+ impersonated: bool = False,) -> None:"""Within an already-held turn slot, register the session in thestream registry, publish the work to the executor queue, and@@ -674,6 +684,7 @@ async def dispatch_turn(message_metadata=message_metadata,envelope=envelope,unattended=unattended,+ impersonated=impersonated,credential_pins=credential_pins,scheduled=scheduled,trace_headers=trace_headers,@@ -707,6 +718,7 @@ async def queue_spawned_turn(permissions: CopilotPermissions | None,spawn: SpawnRequest | None,message_metadata: dict[str, Any] | None,+ impersonated: bool = False,) -> None:"""Queue a turn another session started, to begin when a slot frees.@@ -739,6 +751,7 @@ async def queue_spawned_turn(tool_call_id=tool_call_id,tool_name=tool_name,only_if_idle=True,+ impersonated=impersonated,)except BaseException:await release_turn(envelope)@@ -832,6 +845,7 @@ async def schedule_chat_turn(llm_credential_id: str | None = None,permissions: CopilotPermissions | None = None,request_arrival_at: float = 0.0,+ impersonated: bool = False,) -> str | None:"""HTTP-chat-flavoured :func:`schedule_turn`: acquires the user'sconcurrency slot, persists the user message *inside* the slot, then@@ -915,6 +929,7 @@ async def schedule_chat_turn(llm_credential_id=llm_credential_id,permissions=permissions,request_arrival_at=request_arrival_at,+ impersonated=impersonated,)return turn_iddiff --git a/autogpt_platform/backend/backend/copilot/executor/utils_test.py b/autogpt_platform/backend/backend/copilot/executor/utils_test.pyindex c8a8be32f3..5226b63629 100644--- a/autogpt_platform/backend/backend/copilot/executor/utils_test.py+++ b/autogpt_platform/backend/backend/copilot/executor/utils_test.py@@ -126,6 +126,17 @@ class TestCoPilotExecutionEntry:)assert legacy.unattended is False+ def test_impersonated_survives_the_queue_and_defaults_to_false(self):+ entry = CoPilotExecutionEntry(+ session_id="s1", user_id="u1", message="hi", impersonated=True+ )+ restored = CoPilotExecutionEntry.model_validate_json(entry.model_dump_json())+ assert restored.impersonated is True+ legacy = CoPilotExecutionEntry.model_validate_json(+ '{"session_id": "s1", "user_id": "u1", "message": "hi"}'+ )+ assert legacy.impersonated is False+def test_credential_pins_survive_the_queue_and_default_to_none(self):pin = CredentialPin(id="exa-new", title="Work key")entry = CoPilotExecutionEntry(@@ -267,6 +278,29 @@ async def test_schedule_chat_turn_leaves_a_typed_message_alone() -> None:assert append.await_args.args[1].content == "what did I run yesterday"assert dispatch.await_args.kwargs["message"] == "what did I run yesterday"+ assert dispatch.await_args.kwargs["impersonated"] is False+++async def test_schedule_chat_turn_hands_impersonation_to_the_turn() -> None:+ slot = MagicMock(admitted=True)++ @asynccontextmanager+ async def acquire(*_args, **_kwargs):+ yield slot++ dispatch = AsyncMock()+ with (+ patch.object(utils, "acquire_turn_slot", new=acquire),+ patch("backend.copilot.model.append_and_save_message", new=AsyncMock()),+ patch("backend.copilot.tracking.track_user_message", new=MagicMock()),+ patch.object(utils, "dispatch_turn", new=dispatch),+ ):+ await utils.schedule_chat_turn(+ session_id="s1", user_id="u1", message="hi", impersonated=True+ )++ assert dispatch.await_args.kwargs["impersonated"] is True@pytest.mark.asyncio@@ -363,3 +397,97 @@ class TestNarrowPermissions:assert merged is not Noneassert merged.tools_exclude is Falseassert merged.effective_allowed_tools(ALL_TOOL_NAMES) == {"read_workspace_file"}+++async def test_dispatch_turn_puts_impersonation_on_the_queued_entry() -> None:+ queue = MagicMock(publish_message=AsyncMock())+ with (+ patch.object(+ utils,+ "_admitted_turn_envelope",+ new=AsyncMock(return_value=root_envelope("t1")),+ ),+ patch("backend.copilot.stream_registry.create_session", new=AsyncMock()),+ patch(+ "backend.util.clients.get_async_copilot_queue",+ new=AsyncMock(return_value=queue),+ ),+ ):+ await utils.dispatch_turn(+ MagicMock(),+ session_id="s1",+ user_id="u1",+ turn_id="t1",+ message="hi",+ impersonated=True,+ )++ entry = CoPilotExecutionEntry.model_validate_json(+ queue.publish_message.await_args.kwargs["message"]+ )+ assert entry.impersonated is True+++async def test_schedule_turn_hands_impersonation_to_the_turn() -> None:+ slot = MagicMock(admitted=True)++ @asynccontextmanager+ async def acquire(*_args, **_kwargs):+ yield slot++ dispatch = AsyncMock()+ with (+ patch(+ "backend.copilot.turn_queue.count_inflight_turns",+ new=AsyncMock(return_value=0),+ ),+ patch.object(utils, "acquire_turn_slot", new=acquire),+ patch.object(utils, "dispatch_turn", new=dispatch),+ ):+ await utils.schedule_turn(+ session_id="s1", user_id="u1", turn_id="t1", message="hi"+ )+ assert dispatch.await_args.kwargs["impersonated"] is False++ await utils.schedule_turn(+ session_id="s1",+ user_id="u1",+ turn_id="t2",+ message="hi",+ impersonated=True,+ )+ assert dispatch.await_args.kwargs["impersonated"] is True+++async def test_queue_spawned_turn_stores_impersonation_with_the_queued_row() -> None:+ enqueue = AsyncMock()+ with (+ patch(+ "backend.copilot.turn_queue.count_inflight_turns",+ new=AsyncMock(return_value=0),+ ),+ patch.object(+ utils,+ "_admitted_turn_envelope",+ new=AsyncMock(return_value=root_envelope("t1")),+ ),+ patch("backend.copilot.turn_queue.enqueue_turn", new=enqueue),+ ):+ await utils.queue_spawned_turn(+ session_id="s1",+ user_id="u1",+ message="hi",+ tool_call_id="sub:parent",+ tool_name="run_sub_session",+ llm_auth_provider="platform",+ llm_credential_id=None,+ permissions=None,+ spawn=None,+ message_metadata=None,+ impersonated=True,+ )++ assert enqueue.await_args.kwargs["impersonated"] is Truediff --git a/autogpt_platform/backend/backend/copilot/feedback.py b/autogpt_platform/backend/backend/copilot/feedback.pyindex de565a89e3..10400ccd27 100644--- a/autogpt_platform/backend/backend/copilot/feedback.py+++ b/autogpt_platform/backend/backend/copilot/feedback.py@@ -13,10 +13,20 @@ to Langfuse as a score named after the rating ("user-feedback": 1 = up,The score reuses the rating's id, so rating a reply again updates its scoreinstead of adding one. A Langfuse failure is logged and shows in theresponse; the rating itself is already saved.++Each rating also goes to PostHog LLM analytics as an ``$ai_metric`` event on+the user (distinct id = user id), for every user regardless of cookie+consent, like the generations OpenRouter Broadcast sends there.+``$ai_session_id`` is the chat session id, the same value the chat's+``$ai_generation`` events carry, which is what joins a rating to the+generations of its chat; ``message_id`` names the reply. It carries no+``$ai_trace_id``: Broadcast assigns the generations' trace ids, so we have+none to match. Rating again sends another event; the latest per+``feedback_id`` is the rating that stands."""import logging-from typing import Literal+from typing import Any, Literalfrom langfuse import get_clientfrom langfuse.api import CreateScoreRequest, ScoreDataType@@ -24,6 +34,8 @@ from prisma.models import ChatMessage as PrismaChatMessagefrom pydantic import BaseModel, Fieldfrom backend.copilot import feedback_db+from backend.util.posthog_client import capture+from backend.util.posthog_events import PostHogEventfrom backend.util.settings import Settingslogger = logging.getLogger(__name__)@@ -70,11 +82,14 @@ async def record_message_feedback(score_name: FeedbackScoreName,score_value: FeedbackScoreValue,comment: str | None,+ impersonated: bool = False,) -> MessageFeedbackResponse | None:- """Save a rating of one reply in *user_id*'s chat and score it in Langfuse.+ """Save a rating of one reply in *user_id*'s chat and score it in Langfuse+ and PostHog.*message_id* is the id the chat UI knows the reply by: ``<session>-seq-<N>``for a reply loaded from the chat history, or the message row's own id.+ *impersonated* marks a rating an admin gave while acting as the user.Returns ``None`` when the session is not the caller's or *message_id* isnot a reply in it.@@ -91,6 +106,16 @@ async def record_message_feedback(comment=comment,langfuse_trace_id=message.langfuseTraceId,)+ _send_to_posthog(+ user_id=user_id,+ feedback_id=feedback_id,+ session_id=session_id,+ message=message,+ score_name=score_name,+ score_value=score_value,+ comment=comment,+ impersonated=impersonated,+ )target = await _score_in_langfuse(feedback_id=feedback_id,session_id=session_id,@@ -174,3 +199,31 @@ async def _score_in_langfuse()return Nonereturn "trace" if trace_id else "session"+++def _send_to_posthog(+ *,+ user_id: str,+ feedback_id: str,+ session_id: str,+ message: PrismaChatMessage,+ score_name: FeedbackScoreName,+ score_value: FeedbackScoreValue,+ comment: str | None,+ impersonated: bool,+) -> None:+ """Queue the rating as an ``$ai_metric`` event. ``capture`` logs and+ swallows a PostHog failure, so the rating is never lost to one."""+ properties: dict[str, Any] = {+ "$ai_metric_name": score_name,+ "$ai_metric_value": score_value,+ "$ai_session_id": session_id,+ "chat_session_id": session_id,+ "message_id": message.id,+ "message_sequence": message.sequence,+ "feedback_id": feedback_id,+ "impersonated": impersonated,+ }+ if comment:+ properties["$ai_feedback_text"] = comment+ capture(user_id, PostHogEvent.AI_METRIC, properties)diff --git a/autogpt_platform/backend/backend/copilot/feedback_test.py b/autogpt_platform/backend/backend/copilot/feedback_test.pyindex 92ddc2d59e..556363b4f8 100644--- a/autogpt_platform/backend/backend/copilot/feedback_test.py+++ b/autogpt_platform/backend/backend/copilot/feedback_test.py@@ -1,10 +1,161 @@-"""Tests for reading the chat UI's message ids in ``copilot.feedback``."""+"""Tests for ``copilot.feedback``: reading the chat UI's message ids, and+sending a rating to PostHog LLM analytics.++The route tests (``backend/api/features/chat/feedback_test.py``) cover the+Postgres and Langfuse halves.+"""++from datetime import UTC, datetime+from unittest.mock import AsyncMock, MagicMockimport pytest+import pytest_mock+from prisma.models import ChatMessage as PrismaChatMessage-from backend.copilot.feedback import sequence_from_ui_message_id+from backend.copilot import feedback, feedback_db+from backend.copilot.feedback import (+ record_message_feedback,+ sequence_from_ui_message_id,+)+from backend.util import posthog_client+from backend.util.posthog_events import PostHogEventSESSION_ID = "8d2f5c1e-2b7a-4c55-9d0e-3f6a1b2c4d5e"+USER_ID = "3b1f0e2a-7c4d-4e5f-8a9b-0c1d2e3f4a5b"+MESSAGE_ROW_ID = "0f9e8d7c-6b5a-4938-8271-605f4e3d2c1b"+FEEDBACK_ID = "c3a1b2d4-e5f6-4789-8abc-def012345678"+++def reply(mocker: pytest_mock.MockerFixture) -> PrismaChatMessage:+ message = PrismaChatMessage(+ id=MESSAGE_ROW_ID,+ createdAt=datetime.now(UTC),+ sessionId=SESSION_ID,+ role="assistant",+ content="Here is the report.",+ sequence=7,+ langfuseTraceId="1edf31f11b1693cc6103f358c1481694",+ )+ mocker.patch.object(+ feedback_db, "get_rateable_message", new=AsyncMock(return_value=message)+ )+ mocker.patch.object(+ feedback_db,+ "upsert_message_feedback",+ new=AsyncMock(return_value=FEEDBACK_ID),+ )+ return message+++def langfuse_score(mocker: pytest_mock.MockerFixture) -> AsyncMock:+ secrets = feedback.settings.secrets+ mocker.patch.object(secrets, "langfuse_public_key", "pk-lf-test")+ mocker.patch.object(secrets, "langfuse_secret_key", "sk-lf-test")+ langfuse = MagicMock()+ langfuse.async_api.score.create = AsyncMock()+ mocker.patch.object(feedback, "get_client", return_value=langfuse)+ return langfuse.async_api.score.create+++async def _rate(**overrides):+ kwargs = {+ "user_id": USER_ID,+ "session_id": SESSION_ID,+ "message_id": f"{SESSION_ID}-seq-7",+ "score_name": "user-feedback",+ "score_value": 0,+ "comment": "It edited the wrong file",+ **overrides,+ }+ return await record_message_feedback(**kwargs)++[email protected]("reply")+async def test_thumbs_down_is_an_ai_metric_on_the_user_and_chat(+ mocker: pytest_mock.MockerFixture, langfuse_score: AsyncMock+) -> None:+ capture = mocker.patch.object(feedback, "capture")++ response = await _rate()++ assert response is not None+ capture.assert_called_once_with(+ USER_ID,+ PostHogEvent.AI_METRIC,+ {+ "$ai_metric_name": "user-feedback",+ "$ai_metric_value": 0,+ "$ai_session_id": SESSION_ID,+ "chat_session_id": SESSION_ID,+ "message_id": MESSAGE_ROW_ID,+ "message_sequence": 7,+ "feedback_id": FEEDBACK_ID,+ "impersonated": False,+ "$ai_feedback_text": "It edited the wrong file",+ },+ )+ langfuse_score.assert_awaited_once()+ assert response.langfuse_target == "trace"++[email protected]("reply", "langfuse_score")+async def test_copy_carries_no_feedback_text(+ mocker: pytest_mock.MockerFixture,+) -> None:+ capture = mocker.patch.object(feedback, "capture")++ await _rate(score_name="copy", score_value=1, comment=None)++ properties = capture.call_args.args[2]+ assert properties["$ai_metric_name"] == "copy"+ assert properties["$ai_metric_value"] == 1+ assert "$ai_feedback_text" not in properties++[email protected]("reply", "langfuse_score")+async def test_impersonated_rating_is_marked(+ mocker: pytest_mock.MockerFixture,+) -> None:+ capture = mocker.patch.object(feedback, "capture")++ await _rate(impersonated=True)++ assert capture.call_args.args[2]["impersonated"] is True++[email protected]("reply")+async def test_posthog_failure_keeps_the_rating_and_the_langfuse_score(+ mocker: pytest_mock.MockerFixture, langfuse_score: AsyncMock+) -> None:+ """The real ``capture`` runs against a client that raises."""+ client = MagicMock()+ client.capture.side_effect = RuntimeError("posthog is down")+ mocker.patch.object(posthog_client, "get_posthog_client", return_value=client)++ response = await _rate()++ client.capture.assert_called_once()+ assert client.capture.call_args.kwargs["distinct_id"] == USER_ID+ assert response is not None+ assert response.id == FEEDBACK_ID+ langfuse_score.assert_awaited_once()+++async def test_unknown_reply_sends_nothing(mocker: pytest_mock.MockerFixture) -> None:+ mocker.patch.object(+ feedback_db, "get_rateable_message", new=AsyncMock(return_value=None)+ )+ capture = mocker.patch.object(feedback, "capture")++ assert await _rate() is None+ capture.assert_not_called()@pytest.mark.parametrize(diff --git a/autogpt_platform/backend/backend/copilot/llm_attribution.py b/autogpt_platform/backend/backend/copilot/llm_attribution.pynew file mode 100644index 0000000000..1f555493e8--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/llm_attribution.py@@ -0,0 +1,81 @@+"""Who an AutoPilot LLM call is for, in the fields OpenRouter Broadcast reads.++OpenRouter Broadcast sends every generation to PostHog LLM analytics as an+``$ai_generation`` event (and to Langfuse). It maps these request body fields+(https://openrouter.ai/docs/guides/features/broadcast/posthog):++- ``user`` becomes the event's ``distinct_id``. It is the platform user id,+ the distinct id every other backend and browser event uses, so the+ generation lands on the same person as their product events and replays.+- ``session_id`` becomes ``$ai_session_id``: the chat session id.+- ``trace.generation_name`` becomes ``$ai_span_name``; every other ``trace``+ key arrives as ``metadata_<key>`` (``metadata_environment``,+ ``metadata_impersonated``).++Only calls that go through OpenRouter are covered. Direct Anthropic, Claude+Code subscription and user-connected (Codex) routes never pass OpenRouter, so+Broadcast sees none of their generations.+"""++from contextvars import ContextVar+from typing import Any, Literal++from autogpt_libs.auth.dependencies import IMPERSONATION_HEADER_NAME+from fastapi import Request++from backend.util.settings import Settings++settings = Settings()++# OpenRouter's limit on ``user``. ``session_id`` allows 256; chat session ids+# are UUIDs, so the shorter cap never cuts one.+_MAX_ID_CHARS = 128++GenerationName = Literal["copilot-title", "copilot-sdk", "copilot-baseline"]++# Whether an admin impersonating the user started the running turn. The+# executor sets it for every turn, so a value never carries over.+_turn_impersonated: ContextVar[bool] = ContextVar("_turn_impersonated", default=False)+++def openrouter_attribution(+ *,+ user_id: str | None,+ session_id: str | None,+ generation_name: GenerationName,+) -> dict[str, Any]:+ """The body fields that attribute one OpenRouter request in Broadcast."""+ body: dict[str, Any] = {+ "trace": {+ "generation_name": generation_name,+ "environment": _environment(),+ "impersonated": is_turn_impersonated(),+ }+ }+ if user_id:+ body["user"] = user_id[:_MAX_ID_CHARS]+ if session_id:+ body["session_id"] = session_id[:_MAX_ID_CHARS]+ return body+++def set_turn_impersonated(impersonated: bool) -> None:+ """Mark the running turn as started by an impersonating admin, or not."""+ _turn_impersonated.set(impersonated)+++def is_turn_impersonated() -> bool:+ return _turn_impersonated.get()+++def is_impersonated_request(request: Request) -> bool:+ """Whether an admin made *request* as another user.++ The auth dependencies have already refused the header from anyone but an+ admin by the time a route runs, so its presence is enough.+ """+ return bool(request.headers.get(IMPERSONATION_HEADER_NAME, "").strip())+++def _environment() -> str:+ return settings.config.app_env.valuediff --git a/autogpt_platform/backend/backend/copilot/llm_attribution_test.py b/autogpt_platform/backend/backend/copilot/llm_attribution_test.pynew file mode 100644index 0000000000..02964b6cd6--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/llm_attribution_test.py@@ -0,0 +1,90 @@+"""Tests for the OpenRouter Broadcast attribution fields."""++import asyncio+from unittest.mock import MagicMock++import pytest++from backend.copilot import llm_attribution+from backend.copilot.llm_attribution import (+ is_impersonated_request,+ is_turn_impersonated,+ openrouter_attribution,+ set_turn_impersonated,+)+from backend.util.settings import AppEnvironment++[email protected](autouse=True)+def _not_impersonated():+ set_turn_impersonated(False)+ yield+ set_turn_impersonated(False)+++def test_attribution_names_the_user_chat_and_environment(mocker):+ mocker.patch.object(+ llm_attribution.settings.config, "app_env", AppEnvironment.DEVELOPMENT+ )++ body = openrouter_attribution(+ user_id="user-1", session_id="sess-1", generation_name="copilot-title"+ )++ assert body == {+ "user": "user-1",+ "session_id": "sess-1",+ "trace": {+ "generation_name": "copilot-title",+ "environment": "dev",+ "impersonated": False,+ },+ }+++def test_attribution_without_a_user_or_chat_still_tags_the_environment():+ body = openrouter_attribution(+ user_id=None, session_id=None, generation_name="copilot-baseline"+ )++ assert set(body) == {"trace"}+ assert body["trace"]["generation_name"] == "copilot-baseline"+++def test_attribution_marks_an_impersonated_turn():+ set_turn_impersonated(True)++ body = openrouter_attribution(+ user_id="user-1", session_id="sess-1", generation_name="copilot-sdk"+ )++ assert body["trace"]["impersonated"] is True+++async def test_impersonation_does_not_leak_into_another_turn():+ async def turn(impersonated: bool) -> bool:+ set_turn_impersonated(impersonated)+ await asyncio.sleep(0)+ return is_turn_impersonated()++ admin, user = await asyncio.gather(+ asyncio.create_task(turn(True)), asyncio.create_task(turn(False))+ )++ assert (admin, user) == (True, False)+++ ("headers", "expected"),+ [+ ({"X-Act-As-User-Id": "target-user"}, True),+ ({"X-Act-As-User-Id": " "}, False),+ ({}, False),+ ],+)+def test_is_impersonated_request(headers: dict[str, str], expected: bool):+ request = MagicMock()+ request.headers = headers++ assert is_impersonated_request(request) is expecteddiff --git a/autogpt_platform/backend/backend/copilot/sdk/env.py b/autogpt_platform/backend/backend/copilot/sdk/env.pyindex 2a5979cc30..bed12de8aa 100644--- a/autogpt_platform/backend/backend/copilot/sdk/env.py+++ b/autogpt_platform/backend/backend/copilot/sdk/env.py@@ -8,11 +8,13 @@ circular import through ``executor`` → ``credit`` → ``block_cost_config``).from __future__ import annotations+import jsonimport osimport refrom urllib.parse import urlparsefrom backend.copilot.config import ChatConfig+from backend.copilot.llm_attribution import openrouter_attributionfrom backend.copilot.moonshot import is_moonshot_model, moonshot_context_windowfrom backend.copilot.sdk.subscription import validate_subscription@@ -56,7 +58,10 @@ def build_sdk_env(3. **Direct Anthropic** — subprocess inherits ``ANTHROPIC_API_KEY``from the parent environment (no overrides needed).4. **OpenRouter** (default) — overrides base URL and auth token to- route through the proxy, with Langfuse trace headers.+ route through the proxy, with the headers and body fields OpenRouter+ Broadcast attributes generations by (user, chat session,+ environment). Only this mode reaches Broadcast, so only its+ generations show in PostHog LLM analytics and Langfuse broadcast.All modes receive workspace isolation (``CLAUDE_CODE_TMPDIR``) andsecurity hardening env vars to prevent .claude.md loading, prompt@@ -142,7 +147,9 @@ def build_sdk_env("CLAUDE_CODE_REFRESH_TOKEN": "", # prevent token refresh via subscription}- # Inject broadcast headers so OpenRouter forwards traces to Langfuse.+ # Inject broadcast headers: OpenRouter Broadcast skips the+ # Anthropic-native ``/messages`` endpoint unless ``x-session-id``+ # is present.def _safe(v: str) -> str:return _HEADER_SAFE_RE.sub("", v).strip()[:_MAX_HEADER_VALUE_LEN]@@ -153,6 +160,7 @@ def build_sdk_env(parts.append(f"x-user-id: {_safe(user_id)}")if parts:env["ANTHROPIC_CUSTOM_HEADERS"] = "\n".join(parts)+ env["CLAUDE_CODE_EXTRA_BODY"] = _broadcast_extra_body(session_id, user_id)# --- Common: workspace isolation + security hardening (all modes) ---# Route subagent temp files into the per-session workspace so output@@ -258,3 +266,20 @@ def _autocompact_pct_for_model(model: str | None) -> int:if model and "claude-sonnet-5" in model:pct = min(round(pct * _SONNET_5_TOKENIZER_INFLATION), 90)return pct+++def _broadcast_extra_body(session_id: str | None, user_id: str | None) -> str:+ """``CLAUDE_CODE_EXTRA_BODY``: JSON the CLI merges into every request body+ (main turn, sub-agents, compaction), carrying the fields OpenRouter+ Broadcast attributes a generation by.++ ``metadata.user_id`` replaces the CLI's own (a JSON blob of its device and+ session ids) so that every user field on the request names the same+ platform user.+ """+ body = openrouter_attribution(+ user_id=user_id, session_id=session_id, generation_name="copilot-sdk"+ )+ if user_id:+ body["metadata"] = {"user_id": body["user"]}+ return json.dumps(body)diff --git a/autogpt_platform/backend/backend/copilot/sdk/env_broadcast_test.py b/autogpt_platform/backend/backend/copilot/sdk/env_broadcast_test.pynew file mode 100644index 0000000000..7094f4cf6a--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/sdk/env_broadcast_test.py@@ -0,0 +1,106 @@+"""The OpenRouter Broadcast fields ``build_sdk_env`` hands the Claude CLI.++The CLI owns the request body, so the fields that attribute a generation in+PostHog LLM analytics travel in ``CLAUDE_CODE_EXTRA_BODY``, which the CLI+merges into every request it makes.+"""++import json+from unittest.mock import patch++import pytest++from backend.copilot.config import ChatConfig+from backend.copilot.llm_attribution import set_turn_impersonated+from backend.copilot.llm_attribution import settings as attribution_settings+from backend.copilot.sdk.env import build_sdk_env+++def _config(**overrides) -> ChatConfig:+ fields = {+ "use_claude_code_subscription": False,+ "use_openrouter": True,+ "api_key": "sk-or-test-key",+ "base_url": "https://openrouter.ai/api/v1",+ "thinking_standard_model": "anthropic/claude-sonnet-4-6",+ "thinking_advanced_model": "anthropic/claude-opus-4-7",+ "aux_api_key": "or-aux-key",+ }+ fields.update(overrides)+ return ChatConfig(**fields)++[email protected](autouse=True)+def _not_impersonated():+ set_turn_impersonated(False)+ yield+ set_turn_impersonated(False)+++def _extra_body(**kwargs) -> dict:+ with patch("backend.copilot.sdk.env.config", _config()):+ env = build_sdk_env(**kwargs)+ return json.loads(env["CLAUDE_CODE_EXTRA_BODY"])+++def test_openrouter_mode_attributes_every_request_to_the_user_and_chat():+ body = _extra_body(session_id="sess-1", user_id="user-1")++ assert body == {+ "user": "user-1",+ "session_id": "sess-1",+ "metadata": {"user_id": "user-1"},+ "trace": {+ "generation_name": "copilot-sdk",+ "environment": attribution_settings.config.app_env.value,+ "impersonated": False,+ },+ }+++def test_openrouter_mode_keeps_the_session_header_broadcast_needs():+ with patch("backend.copilot.sdk.env.config", _config()):+ env = build_sdk_env(session_id="sess-1", user_id="user-1")++ assert "x-session-id: sess-1" in env["ANTHROPIC_CUSTOM_HEADERS"]+++def test_an_impersonated_turn_is_marked():+ set_turn_impersonated(True)++ body = _extra_body(session_id="sess-1", user_id="user-1")++ assert body["trace"]["impersonated"] is True+++def test_no_user_sends_no_user_fields():+ body = _extra_body(session_id="sess-1")++ assert "user" not in body+ assert "metadata" not in body+ assert body["session_id"] == "sess-1"+++def test_long_ids_are_cut_to_openrouters_limit():+ body = _extra_body(session_id="s" * 300, user_id="u" * 300)++ assert body["user"] == "u" * 128+ assert body["metadata"] == {"user_id": "u" * 128}+ assert body["session_id"] == "s" * 128+++def test_direct_anthropic_mode_sends_no_broadcast_fields():+ """Broadcast never sees a call that skips OpenRouter."""+ with patch("backend.copilot.sdk.env.config", _config(use_openrouter=False)):+ env = build_sdk_env(session_id="sess-1", user_id="user-1")++ assert "CLAUDE_CODE_EXTRA_BODY" not in env+++@patch("backend.copilot.sdk.env.validate_subscription")+def test_subscription_mode_sends_no_broadcast_fields(_validate):+ cfg = _config(use_claude_code_subscription=True)+ with patch("backend.copilot.sdk.env.config", cfg):+ env = build_sdk_env(session_id="sess-1", user_id="user-1")++ assert "CLAUDE_CODE_EXTRA_BODY" not in envdiff --git a/autogpt_platform/backend/backend/copilot/sdk/session_waiter.py b/autogpt_platform/backend/backend/copilot/sdk/session_waiter.pyindex b1251f8b10..621ffb61c4 100644--- a/autogpt_platform/backend/backend/copilot/sdk/session_waiter.py+++ b/autogpt_platform/backend/backend/copilot/sdk/session_waiter.py@@ -28,6 +28,7 @@ from backend.copilot import stream_registryfrom backend.copilot.active_turns import ConcurrentTurnLimitErrorfrom backend.copilot.config import CopilotLlmAuthProviderfrom backend.copilot.executor.utils import queue_spawned_turn, schedule_turn+from backend.copilot.llm_attribution import is_turn_impersonatedfrom backend.copilot.model import (CHAT_STATUS_QUEUED,_get_session_lock,@@ -278,6 +279,8 @@ async def run_copilot_turn_via_queue(spawn=spawn,message_metadata=message_metadata,delegated=delegated,+ # A child of an impersonated turn is the admin's work too.+ impersonated=is_turn_impersonated(),)except TreeRefusal as refused:return "refused", SessionResult(refusal=refused.message)@@ -369,6 +372,7 @@ async def _queue_for_slot(permissions=permissions,spawn=spawn,message_metadata=message_metadata,+ impersonated=is_turn_impersonated(),)except TreeRefusal as refused:return "refused", SessionResult(refusal=refused.message)diff --git a/autogpt_platform/backend/backend/copilot/sdk/session_waiter_test.py b/autogpt_platform/backend/backend/copilot/sdk/session_waiter_test.pyindex d1018bb798..26c67b839b 100644--- a/autogpt_platform/backend/backend/copilot/sdk/session_waiter_test.py+++ b/autogpt_platform/backend/backend/copilot/sdk/session_waiter_test.py@@ -446,3 +446,77 @@ def _status_db(status: str) -> MagicMock:db = MagicMock()db.get_chat_session_status = AsyncMock(return_value=status)return db++[email protected]("impersonated", [False, True])+async def test_a_child_turn_carries_its_parents_impersonation(impersonated: bool):+ """A sub-session started inside an impersonated turn is the admin's work+ too, so its LLM analytics are marked the same way."""+ scheduled = AsyncMock()++ with (+ patch(+ "backend.copilot.sdk.session_waiter.is_turn_in_flight",+ new=AsyncMock(return_value=False),+ ),+ patch(+ "backend.copilot.sdk.session_waiter.is_turn_impersonated",+ return_value=impersonated,+ ),+ patch("backend.copilot.sdk.session_waiter.schedule_turn", new=scheduled),+ patch(+ "backend.copilot.sdk.session_waiter.wait_for_session_result",+ new=AsyncMock(return_value=("completed", SessionResult())),+ ),+ ):+ await run_copilot_turn_via_queue(+ session_id="sess-sub",+ user_id="u1",+ message="spawn",+ timeout=5,+ tool_call_id="sub:parent",+ tool_name="run_sub_session",+ )++ assert scheduled.await_args.kwargs["impersonated"] is impersonated+++async def test_a_queued_child_turn_carries_its_parents_impersonation(+ mock_session_lookup,+):+ """A delegated child the running cap sends to the queue keeps the flag+ for when it is promoted."""+ mock_session_lookup.return_value.metadata.delegated_by_session_id = "parent"+ queued = AsyncMock()++ with (+ patch(+ "backend.copilot.sdk.session_waiter.is_turn_in_flight",+ new=AsyncMock(return_value=False),+ ),+ patch(+ "backend.copilot.sdk.session_waiter.is_turn_impersonated",+ return_value=True,+ ),+ patch(+ "backend.copilot.sdk.session_waiter.schedule_turn",+ new=AsyncMock(side_effect=active_turns.ConcurrentTurnLimitError()),+ ),+ patch("backend.copilot.sdk.session_waiter.queue_spawned_turn", new=queued),+ patch(+ "backend.copilot.sdk.session_waiter.wait_for_queued_session",+ new=AsyncMock(return_value=("completed", SessionResult())),+ ),+ ):+ await run_copilot_turn_via_queue(+ session_id="sess-sub",+ user_id="u1",+ message="spawn",+ timeout=5,+ tool_call_id="sub:parent",+ tool_name="run_sub_session",+ )++ assert queued.await_args.kwargs["impersonated"] is Truediff --git a/autogpt_platform/backend/backend/copilot/service.py b/autogpt_platform/backend/backend/copilot/service.pyindex c7d90266af..9710de55a2 100644--- a/autogpt_platform/backend/backend/copilot/service.py+++ b/autogpt_platform/backend/backend/copilot/service.py@@ -36,6 +36,7 @@ from .config import ChatConfig, CopilotLLMModelfrom .expert_context import OWNED_BLOCK_TAGS as _EXPERT_BLOCK_TAGSfrom .expert_context import build_expert_context, escape_prompt_xml_tagsfrom .expert_kickoff import is_expert_kickoff_message+from .llm_attribution import openrouter_attributionfrom .model import (ChatMessage,ChatSessionInfo,@@ -827,24 +828,24 @@ async def _generate_session_title(``create`` did not bill, so there is no cost to record."""try:- # Build extra_body for OpenRouter tracing and PostHog analytics.+ # Build extra_body for OpenRouter cost and Broadcast attribution.# ``usage: {"include": True}`` asks OR to embed the real billed# cost into the final usage chunk — matches the baseline path's# ``_OPENROUTER_INCLUDE_USAGE_COST`` pattern, same read path.# Gated on the aux transport because Anthropic's OpenAI-compat# endpoint (and any non-OR endpoint) rejects unknown extra_body# fields with a 400 — the same gate the baseline path applies.+ # Off OpenRouter, Broadcast never sees the call.extra_body: dict[str, Any] = {}if config.aux_uses_openrouter:extra_body["usage"] = {"include": True}- if user_id:- extra_body["user"] = user_id[:128] # OpenRouter limit- extra_body["posthogDistinctId"] = user_id- if session_id:- extra_body["session_id"] = session_id[:128] # OpenRouter limit- extra_body["posthogProperties"] = {- "environment": settings.config.app_env.value,- }+ extra_body.update(+ openrouter_attribution(+ user_id=user_id,+ session_id=session_id,+ generation_name="copilot-title",+ )+ )# Normalize the title model for the aux client's transport: OR# routes by full ``vendor/model`` slug, but Anthropic'sdiff --git a/autogpt_platform/backend/backend/copilot/service_unit_test.py b/autogpt_platform/backend/backend/copilot/service_unit_test.pyindex 23e3877987..e9b66293ef 100644--- a/autogpt_platform/backend/backend/copilot/service_unit_test.py+++ b/autogpt_platform/backend/backend/copilot/service_unit_test.py@@ -28,6 +28,7 @@ from backend.copilot.service import (_record_title_generation_cost,_title_usage_from_response,_update_title_async,+ settings,)from backend.util.settings import Config@@ -667,6 +668,36 @@ class TestGenerateSessionTitle:assert extra_body["usage"] == {"include": True}assert extra_body["user"] == "user-abc"assert extra_body["session_id"] == "sess-abc"+ assert extra_body["trace"] == {+ "generation_name": "copilot-title",+ "environment": settings.config.app_env.value,+ "impersonated": False,+ }++ @pytest.mark.asyncio+ async def test_title_call_off_openrouter_carries_no_broadcast_fields(self):+ """Broadcast only sees OpenRouter calls, and a direct-Anthropic aux+ endpoint 400s on unknown body fields."""+ resp = _build_completion(content="Title")+ helper = AsyncMock(return_value=resp)+ with (+ patch(+ "backend.copilot.service.call_provider_openai_compat_sync",+ new=helper,+ ),+ patch(+ "backend.copilot.service.config",+ MagicMock(+ aux_uses_openrouter=False,+ aux_provider_label="anthropic",+ title_model="anthropic/claude-haiku-4.5",+ ),+ ),+ ):+ await _generate_session_title(+ "hello", user_id="user-abc", session_id="sess-abc"+ )+ assert helper.await_args.kwargs["extra_body"] is None@pytest.mark.asyncioasync def test_title_model_normalized_when_aux_is_anthropic(self):diff --git a/autogpt_platform/backend/backend/copilot/trial_reads_test.py b/autogpt_platform/backend/backend/copilot/trial_reads_test.pyindex 9f0608dec5..aac2fdacea 100644--- a/autogpt_platform/backend/backend/copilot/trial_reads_test.py+++ b/autogpt_platform/backend/backend/copilot/trial_reads_test.py@@ -152,7 +152,11 @@ async def test_the_stream_route_runs_its_checks_in_one_block():pytest.raises(_StopTheTurn),):await stream_chat_post(- "session-1", StreamChatRequest(message="hi"), "u1", MagicMock()+ "session-1",+ StreamChatRequest(message="hi"),+ "u1",+ MagicMock(),+ http_request=MagicMock(),)assert read.await_count == 1diff --git a/autogpt_platform/backend/backend/copilot/turn_queue.py b/autogpt_platform/backend/backend/copilot/turn_queue.pyindex 48768c3987..11e8770d66 100644--- a/autogpt_platform/backend/backend/copilot/turn_queue.py+++ b/autogpt_platform/backend/backend/copilot/turn_queue.py@@ -10,7 +10,8 @@ text column:The user's pending message itself is just a normal ChatMessage row (nostatus of its own). The dispatcher's submit-time payload (``file_ids``,-``model``, ``permissions``, ``context``, ``request_arrival_at``)+``model``, ``permissions``, ``context``, ``request_arrival_at``,+``impersonated``)is stashed in that row's ``metadata`` JSONB so a later promotion canreplay the turn faithfully.@@ -148,6 +149,7 @@ async def try_enqueue_turn(permissions: Mapping[str, Any] | None = None,request_arrival_at: float = 0.0,envelope: TurnEnvelope | None = None,+ impersonated: bool = False,) -> ChatMessage | None:"""Admit a queued turn against the user's hard cap.@@ -174,6 +176,7 @@ async def try_enqueue_turn(permissions=permissions,request_arrival_at=request_arrival_at,envelope=envelope,+ impersonated=impersonated,)@@ -196,6 +199,7 @@ async def enqueue_turn(tool_call_id: str | None = None,tool_name: str | None = None,only_if_idle: bool = False,+ impersonated: bool = False,) -> ChatMessage | None:"""Persist the user's pending message and flip the session to``"queued"``. Caller is responsible for the in-flight cap check@@ -220,6 +224,8 @@ async def enqueue_turn(metadata["permissions"] = dict(permissions)if request_arrival_at:metadata["request_arrival_at"] = request_arrival_at+ if impersonated:+ metadata["impersonated"] = True# A wake starts under the envelope its call was held under, and a spawned# child under the one admitted for it, whichever turn frees the slot.if envelope is not None:@@ -514,6 +520,7 @@ async def _promote_head(user_id: str) -> bool | None:llm_credential_id=head.metadata.llm_credential_id,permissions=_promotion_permissions(head, metadata, spawned=spawned),request_arrival_at=float(metadata.get("request_arrival_at") or 0.0),+ impersonated=metadata.get("impersonated") is True,tool_call_id=metadata.get("tool_call_id") or "chat_stream",tool_name=metadata.get("tool_name") or "chat",# A typed message is a root, as the chat route makes it, not thediff --git a/autogpt_platform/backend/backend/copilot/turn_queue_test.py b/autogpt_platform/backend/backend/copilot/turn_queue_test.pyindex d7b46b3530..b9a5014d41 100644--- a/autogpt_platform/backend/backend/copilot/turn_queue_test.py+++ b/autogpt_platform/backend/backend/copilot/turn_queue_test.py@@ -135,6 +135,27 @@ async def test_enqueue_turn_packs_metadata_into_metadata_payload() -> None:)+async def test_enqueue_turn_records_an_impersonated_turn() -> None:+ db = MagicMock()+ db.get_next_sequence = AsyncMock(return_value=1)+ db.add_chat_message = AsyncMock(return_value=_pyd_message())+ db.update_chat_session_status = AsyncMock(return_value=True)+ with (+ patch.object(turn_queue, "chat_db", return_value=db),+ patch.object(turn_queue, "count_inflight_turns", new=AsyncMock(return_value=0)),+ patch.object(turn_queue, "invalidate_session_cache", new=AsyncMock()),+ ):+ await turn_queue.try_enqueue_turn(+ user_id="u1",+ inflight_cap=15,+ session_id="s1",+ message="hello",+ impersonated=True,+ )+ assert db.add_chat_message.call_args.kwargs["metadata"]["impersonated"] is True++@pytest.mark.asyncioasync def test_enqueue_turn_only_includes_default_transport_without_extra_params() -> (None@@ -1441,11 +1462,16 @@ async def test_dispatch_defers_on_rate_limit_unavailable() -> None:@pytest.mark.asyncio-async def test_dispatch_happy_path_claims_and_dispatches() -> None:[email protected]("impersonated", [False, True])+async def test_dispatch_happy_path_claims_and_dispatches(impersonated: bool) -> None:"""All gates pass → claim session queued → running, build a TurnSlot,- dispatch_turn, invalidate cache, return True."""+ dispatch_turn, invalidate cache, return True. A turn queued by an+ impersonating admin is promoted as one."""head = _queued_row(session_id="s1")- pending = _pyd_message(metadata={"mode": "extended_thinking"})+ stored: dict = {"mode": "extended_thinking"}+ if impersonated:+ stored["impersonated"] = True+ pending = _pyd_message(metadata=stored)db = MagicMock()db.admit_chat_session_turn = AsyncMock(return_value="admitted")db.update_chat_session_status = AsyncMock(return_value=True)@@ -1476,6 +1502,7 @@ async def test_dispatch_happy_path_claims_and_dispatches() -> None:promoted = await turn_queue.dispatch_next_for_user("u1")assert promoted is Truedispatch_turn_mock.assert_awaited_once()+ assert dispatch_turn_mock.await_args.kwargs["impersonated"] is impersonatedinvalidate.assert_awaited_once_with("s1")# Claimed queued → running under the user's cap; nothing restored.assert db.admit_chat_session_turn.await_args.kwargs == {diff --git a/autogpt_platform/backend/backend/util/posthog_events.py b/autogpt_platform/backend/backend/util/posthog_events.pyindex 01ab56099c..f550ebe770 100644--- a/autogpt_platform/backend/backend/util/posthog_events.py+++ b/autogpt_platform/backend/backend/util/posthog_events.py@@ -68,6 +68,11 @@ class PostHogEvent(StrEnum):# event, not a user action; keep it out of funnels.SET_PERSON_PROPERTIES = "$set"+ # LLM analytics (SECRT-2730): backend/copilot/feedback.py. PostHog's own+ # event for a score on a generative AI trace, named by PostHog rather+ # than the analytics plan; a thumbs up/down or copy of an AutoPilot reply.+ AI_METRIC = "$ai_metric"+# Key moments (SECRT-2723): backend/util/product_analytics.pySIGNUP_COMPLETED = "signup_completed"ONBOARDING_COMPLETED = "onboarding_completed"diff --git a/autogpt_platform/backend/backend/util/posthog_events_test.py b/autogpt_platform/backend/backend/util/posthog_events_test.pyindex 1cca68c21c..5e6e533713 100644--- a/autogpt_platform/backend/backend/util/posthog_events_test.py+++ b/autogpt_platform/backend/backend/util/posthog_events_test.py@@ -41,6 +41,7 @@ LIVE_EVENT_NAMES = {"trial_converted","payment_failed","$set",+ "$ai_metric","signup_completed","onboarding_completed","checkout_started",diff --git a/autogpt_platform/frontend/src/app/api/openapi.json b/autogpt_platform/frontend/src/app/api/openapi.jsonindex 4fd7d69f7c..b28802a1bc 100644--- a/autogpt_platform/frontend/src/app/api/openapi.json+++ b/autogpt_platform/frontend/src/app/api/openapi.json@@ -4428,7 +4428,7 @@"post": {"tags": ["v2", "chat", "chat"],"summary": "Submit message feedback",- "description": "Rate a reply in one of the caller's chats.\n\nSaves the rating, one per reply and score name so rating again\noverwrites it, and scores it in Langfuse on the trace of the turn that\nwrote the reply, or on the chat's Langfuse session when that trace is\nnot known. Langfuse being down does not fail the request; the response\nsays where the score went.",+ "description": "Rate a reply in one of the caller's chats.\n\nSaves the rating, one per reply and score name so rating again\noverwrites it, and scores it in Langfuse on the trace of the turn that\nwrote the reply, or on the chat's Langfuse session when that trace is\nnot known. It also goes to PostHog LLM analytics as an ``$ai_metric``\nevent. Langfuse or PostHog being down does not fail the request; the\nresponse says where the Langfuse score went.","operationId": "postV2Submit message feedback","security": [{ "HTTPBearerJWT": [] }],"parameters": [diff --git a/docs/engineering/tracking-plan.md b/docs/engineering/tracking-plan.mdindex 50ded17b63..f8c04cdc8b 100644--- a/docs/engineering/tracking-plan.md+++ b/docs/engineering/tracking-plan.md@@ -298,6 +298,45 @@ sent or dropped this way, not while it is held.Return visits are `$pageview`; account-level retention is computed in`retention_task_weekly` and `user_lifecycle`, not sent as events.+## LLM analytics++AutoPilot's LLM calls and the ratings of its replies, in PostHog's LLM+analytics (SECRT-2730). Both are sent server-side for every user, whatever+their cookie consent, like every backend event. The event names are+PostHog's own, not the analytics plan's.++| Event | Sender | Status | Required properties | Fires when |+| --- | --- | --- | --- | --- |+| `$ai_generation` | OpenRouter Broadcast | live | set by OpenRouter from the request (`backend/copilot/llm_attribution.py`): `distinct_id` = the request's `user` (the platform user id), `$ai_session_id` = its `session_id` (the chat session id), `$ai_span_name` = `trace.generation_name` (`copilot-title`, `copilot-sdk`, `copilot-baseline`), `metadata_environment` (`local`, `dev`, `prod`), `metadata_impersonated`; model, tokens, cost and latency from OpenRouter | An AutoPilot LLM call goes through OpenRouter: a session title, an SDK turn (main agent, sub-agents and the CLI's compaction, via `CLAUDE_CODE_EXTRA_BODY`) or a baseline turn. |+| `$ai_metric` | backend | live | `$ai_metric_name` (`user-feedback`, `copy`), `$ai_metric_value` (`1` thumbs up or copy, `0` thumbs down), `$ai_session_id` and `chat_session_id` (the chat session id), `message_id` (the reply's row id), `message_sequence`, `feedback_id` (the stored rating, shared with its Langfuse score), `impersonated`; `$ai_feedback_text` when a thumbs-down has a comment | A reply is rated (`POST /api/chat/sessions/{id}/feedback`, `copilot/feedback.py`). Rating a reply again sends another event; the latest per `feedback_id` stands. |++What joins them:++- **Person.** A generation's `distinct_id` and a rating's are both the+ platform user id, the id every other event and `posthog.identify` use, so+ both land on the person with their product events and replays.+- **Chat.** `$ai_session_id` is the chat session id on both, so a chat's+ generations and its ratings group in the Sessions tab and join on it.+ `message_id` says which reply was rated; a rating carries no+ `$ai_trace_id`, since Broadcast assigns the generations' trace ids.+- **Environment.** OpenRouter forwards our `trace` keys as `metadata_<key>`,+ so a generation's environment is `metadata_environment`, not the+ `environment` base property; `$ai_metric` has both base properties as usual.+- **Impersonation.** A turn or rating an admin made while impersonating the+ user (`X-Act-As-User-Id`) is still sent, as the user, with+ `metadata_impersonated` / `impersonated` true, so it can be filtered out.+ The mark is kept for a turn queued behind the running-turn cap and for+ sub-sessions such a turn starts. A follow-up sent while a turn runs takes+ that turn's mark, and an engine-switch continuation or an approval wake+ starts unmarked.++Calls that skip OpenRouter never reach Broadcast and have no+`$ai_generation`: direct Anthropic and Claude Code subscription deployments,+a user's connected Codex account and the local transport. Auxiliary calls+that are not attributed yet (the baseline path's context compaction, the+action gate's classifiers, web search, Graphiti memory) arrive without the+user or chat when their client goes through OpenRouter.+## Experiments| Event | Sender | Status | Required properties | Fires when |@@ -483,5 +522,5 @@ so nobody adds a duplicate:- `$pageleave`, `$autocapture`, `$identify` and `$feature_flag_called`:posthog-js (`capture_pageleave` and `autocapture` are on).-- `$ai_generation`: OpenRouter's PostHog integration, for copilot title- generation (`posthogDistinctId` in `copilot/service.py`).+- `$ai_generation`: OpenRouter Broadcast, for every AutoPilot LLM call that+ goes through OpenRouter (see [LLM analytics](#llm-analytics)).