Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

feat(backend/copilot): attribute AutoPilot generations in PostHog and send reply feedback there (#15498)

Krzysztof Czerwinski40d83480a221
diff --git a/autogpt_platform/backend/backend/api/features/chat/feedback.py b/autogpt_platform/backend/backend/api/features/chat/feedback.py
index d04f06f8a2..7bea52fead 100644
--- a/autogpt_platform/backend/backend/api/features/chat/feedback.py
+++ b/autogpt_platform/backend/backend/api/features/chat/feedback.py
@@ -7,7 +7,7 @@ logic lives in ``backend.copilot.feedback``.
from typing import Annotated
from autogpt_libs import auth
-from fastapi import APIRouter, HTTPException, Security
+from fastapi import APIRouter, HTTPException, Request, Security
from pydantic import BaseModel, Field, field_validator, model_validator
from backend.copilot.feedback import (
@@ -17,6 +17,7 @@ from backend.copilot.feedback import (
MessageFeedbackResponse,
record_message_feedback,
)
+from backend.copilot.llm_attribution import is_impersonated_request
router = APIRouter(tags=["chat"])
@@ -71,14 +72,16 @@ async def submit_message_feedback(
session_id: str,
request: MessageFeedbackRequest,
user_id: Annotated[str, Security(auth.get_user_id)],
+ http_request: Request,
) -> MessageFeedbackResponse:
"""Rate a reply in one of the caller's chats.
Saves the rating, one per reply and score name so rating again
overwrites it, and scores it in Langfuse on the trace of the turn that
wrote the reply, or on the chat's Langfuse session when that trace is
- not known. Langfuse being down does not fail the request; the response
- says where the score went.
+ not known. It also goes to PostHog LLM analytics as an ``$ai_metric``
+ event. Langfuse or PostHog being down does not fail the request; the
+ response says where the Langfuse score went.
"""
feedback = await record_message_feedback(
user_id=user_id,
@@ -87,6 +90,7 @@ async def submit_message_feedback(
score_name=request.score_name,
score_value=request.score_value,
comment=request.comment,
+ impersonated=is_impersonated_request(http_request),
)
if feedback is None:
raise HTTPException(
diff --git a/autogpt_platform/backend/backend/api/features/chat/feedback_test.py b/autogpt_platform/backend/backend/api/features/chat/feedback_test.py
index fc88c67cc8..608e4aa3c2 100644
--- a/autogpt_platform/backend/backend/api/features/chat/feedback_test.py
+++ b/autogpt_platform/backend/backend/api/features/chat/feedback_test.py
@@ -12,6 +12,7 @@ import fastapi
import fastapi.testclient
import pytest
import pytest_mock
+from autogpt_libs.auth.jwt_utils import get_jwt_payload
from langfuse.api import ScoreDataType
from prisma.models import ChatMessage as PrismaChatMessage
@@ -31,8 +32,6 @@ FEEDBACK_ID = "c3a1b2d4-e5f6-4789-8abc-def012345678"
@pytest.fixture(autouse=True)
def setup_app_auth(mock_jwt_user):
- from autogpt_libs.auth.jwt_utils import get_jwt_payload
-
app.dependency_overrides[get_jwt_payload] = mock_jwt_user["get_jwt_payload"]
yield
app.dependency_overrides.clear()
@@ -46,6 +45,13 @@ def langfuse_settings(mocker: pytest_mock.MockerFixture):
mocker.patch.object(secrets, "langfuse_tracing_environment", "dev")
[email protected](autouse=True)
+def posthog_capture(mocker: pytest_mock.MockerFixture) -> MagicMock:
+ """PostHog is covered in ``backend/copilot/feedback_test.py``; here it
+ only must not reach a real client."""
+ return mocker.patch.object(feedback_service, "capture")
+
+
@pytest.fixture
def score_create(mocker: pytest_mock.MockerFixture) -> AsyncMock:
"""The Langfuse score API call, with the client it hangs off mocked."""
@@ -355,6 +361,48 @@ def test_invalid_ratings_are_rejected(
upsert.assert_not_awaited()
+def test_rating_is_sent_to_posthog_as_the_user(
+ mocker: pytest_mock.MockerFixture,
+ test_user_id: str,
+ upsert: AsyncMock,
+ score_create: AsyncMock,
+ posthog_capture: MagicMock,
+) -> None:
+ _mock_message(mocker, _reply(trace_id=TRACE_ID))
+
+ response = _post(_thumbs_down())
+
+ assert response.status_code == 200
+ distinct_id, event, properties = posthog_capture.call_args.args
+ assert distinct_id == test_user_id
+ assert event == "$ai_metric"
+ assert properties["impersonated"] is False
+ score_create.assert_awaited_once()
+
+
+def test_rating_by_an_impersonating_admin_is_marked(
+ mocker: pytest_mock.MockerFixture,
+ mock_jwt_admin,
+ target_user_id: str,
+ upsert: AsyncMock,
+ score_create: AsyncMock,
+ posthog_capture: MagicMock,
+) -> None:
+ app.dependency_overrides[get_jwt_payload] = mock_jwt_admin["get_jwt_payload"]
+ _mock_message(mocker, _reply(trace_id=TRACE_ID))
+
+ response = client.post(
+ f"/sessions/{SESSION_ID}/feedback",
+ json=_thumbs_down(),
+ headers={"X-Act-As-User-Id": target_user_id},
+ )
+
+ assert response.status_code == 200
+ distinct_id, _, properties = posthog_capture.call_args.args
+ assert distinct_id == target_user_id
+ assert properties["impersonated"] is True
+
+
def test_blank_comment_is_saved_as_none(
mocker: pytest_mock.MockerFixture,
upsert: AsyncMock,
diff --git a/autogpt_platform/backend/backend/api/features/chat/routes.py b/autogpt_platform/backend/backend/api/features/chat/routes.py
index 209d87a987..8563374685 100644
--- a/autogpt_platform/backend/backend/api/features/chat/routes.py
+++ b/autogpt_platform/backend/backend/api/features/chat/routes.py
@@ -8,7 +8,7 @@ from typing import Annotated, Any, Literal
from uuid import uuid4
from autogpt_libs import auth
-from fastapi import APIRouter, HTTPException, Query, Response, Security
+from fastapi import APIRouter, HTTPException, Query, Request, Response, Security
from fastapi.responses import JSONResponse, StreamingResponse
from pydantic import BaseModel, ConfigDict, Field, field_validator
@@ -41,6 +41,7 @@ from backend.copilot.expert_kickoff import (
expert_kickoff_metadata,
scoped_client_message_id,
)
+from backend.copilot.llm_attribution import is_impersonated_request
from backend.copilot.model import (
CHAT_STATUS_IDLE,
CHAT_STATUS_RUNNING,
@@ -1750,6 +1751,8 @@ async def stream_chat_post(
request: StreamChatRequest,
user_id: str = Security(auth.get_user_id),
ctx: auth.RequestContext = Security(auth.get_request_context),
+ *,
+ http_request: Request,
):
"""Start a new turn and return an AI SDK UI message stream.
@@ -2044,6 +2047,7 @@ async def stream_chat_post(
llm_credential_id=session.metadata.llm_credential_id,
permissions=builder_permissions,
request_arrival_at=request_arrival_at,
+ impersonated=is_impersonated_request(http_request),
)
except ConcurrentTurnLimitError as exc:
if resume_persisted_kickoff:
@@ -2074,6 +2078,7 @@ async def stream_chat_post(
else None
),
request_arrival_at=request_arrival_at,
+ impersonated=is_impersonated_request(http_request),
)
except turn_queue.InflightCapExceeded:
await release_client_message_claim()
diff --git a/autogpt_platform/backend/backend/api/features/chat/routes_test.py b/autogpt_platform/backend/backend/api/features/chat/routes_test.py
index 4ab3d61c54..7f7cab322f 100644
--- a/autogpt_platform/backend/backend/api/features/chat/routes_test.py
+++ b/autogpt_platform/backend/backend/api/features/chat/routes_test.py
@@ -11,6 +11,7 @@ import fastapi.testclient
import httpx
import pytest
import pytest_mock
+from autogpt_libs.auth.jwt_utils import get_jwt_payload
from pydantic import SecretStr
from sentry_sdk.integrations.asgi import SentryAsgiMiddleware
@@ -73,7 +74,6 @@ def test_tool_response_union_exports_expert_soul_response() -> None:
def setup_app_auth(mock_jwt_user, mocker: pytest_mock.MockerFixture):
"""Setup auth overrides for all tests in this module"""
from autogpt_libs.auth.dependencies import get_request_context
- from autogpt_libs.auth.jwt_utils import get_jwt_payload
app.dependency_overrides[get_jwt_payload] = mock_jwt_user["get_jwt_payload"]
app.dependency_overrides[get_request_context] = mock_jwt_user["get_request_context"]
@@ -431,6 +431,78 @@ def test_stream_chat_accepts_20_file_ids(mocker: pytest_mock.MockerFixture):
assert response.status_code == 200
+def _mock_rate_limits_for_an_unseeded_user(mocker: pytest_mock.MockerFixture) -> None:
+ """The impersonated user has no row in the test database, so the tier
+ lookup behind the rate limit would fail before the turn is scheduled."""
+ mocker.patch(
+ "backend.api.features.chat.routes.get_global_rate_limits",
+ new_callable=AsyncMock,
+ return_value=(0, 0, SubscriptionTier.BASIC),
+ )
+ mocker.patch(
+ "backend.api.features.chat.routes.check_rate_limit", new_callable=AsyncMock
+ )
+
+
+def test_stream_chat_turn_is_not_impersonated_by_default(
+ mocker: pytest_mock.MockerFixture,
+) -> None:
+ mocks = _mock_stream_internals(mocker)
+
+ response = client.post("/sessions/sess-1/stream", json={"message": "hello"})
+
+ assert response.status_code == 200
+ assert mocks.enqueue.await_args.kwargs["impersonated"] is False
+
+
+def test_stream_chat_marks_a_turn_an_impersonating_admin_starts(
+ mocker: pytest_mock.MockerFixture,
+ mock_jwt_admin,
+ target_user_id: str,
+) -> None:
+ """The turn still runs and is still tracked, as the user, but its LLM
+ analytics say an admin sent it."""
+ app.dependency_overrides[get_jwt_payload] = mock_jwt_admin["get_jwt_payload"]
+ mocks = _mock_stream_internals(mocker)
+ _mock_rate_limits_for_an_unseeded_user(mocker)
+
+ response = client.post(
+ "/sessions/sess-1/stream",
+ json={"message": "hello"},
+ headers={"X-Act-As-User-Id": target_user_id},
+ )
+
+ assert response.status_code == 200
+ kwargs = mocks.enqueue.await_args.kwargs
+ assert kwargs["user_id"] == target_user_id
+ assert kwargs["impersonated"] is True
+
+
+def test_stream_chat_queues_an_impersonated_turn_as_impersonated(
+ mocker: pytest_mock.MockerFixture,
+ mock_jwt_admin,
+ target_user_id: str,
+) -> None:
+ """A turn queued behind the running cap is promoted later, from the stored
+ row, so the row has to say an admin sent it."""
+ app.dependency_overrides[get_jwt_payload] = mock_jwt_admin["get_jwt_payload"]
+ mocks = _mock_stream_internals(mocker)
+ _mock_rate_limits_for_an_unseeded_user(mocker)
+ mocks.enqueue.side_effect = chat_routes.ConcurrentTurnLimitError("busy")
+ queue = mocker.patch.object(
+ chat_routes.turn_queue, "try_enqueue_turn", new_callable=AsyncMock
+ )
+
+ client.post(
+ "/sessions/sess-1/stream",
+ json={"message": "hello"},
+ headers={"X-Act-As-User-Id": target_user_id},
+ )
+
+ assert queue.await_args.kwargs["user_id"] == target_user_id
+ assert queue.await_args.kwargs["impersonated"] is True
+
+
def test_stream_chat_allows_an_active_expert_session(
mocker: pytest_mock.MockerFixture,
test_user_id: str,
diff --git a/autogpt_platform/backend/backend/copilot/baseline/service.py b/autogpt_platform/backend/backend/copilot/baseline/service.py
index fe44f3ea9e..d44719a3a9 100644
--- a/autogpt_platform/backend/backend/copilot/baseline/service.py
+++ b/autogpt_platform/backend/backend/copilot/baseline/service.py
@@ -55,6 +55,7 @@ from backend.copilot.gate.held import resolve_answered
from backend.copilot.graphiti.config import is_enabled_for_user
from backend.copilot.graphiti.context import fetch_warm_context
from backend.copilot.graphiti.ingest import enqueue_conversation_turn
+from backend.copilot.llm_attribution import openrouter_attribution
from backend.copilot.local_context_probe import (
compaction_target_for_window,
probe_local_context_window,
@@ -510,6 +511,10 @@ class _BaselineStreamState:
# what already ran; see backend/copilot/segments.py.
llm_auth_provider: CopilotLlmAuthProvider | None = None
llm_credential_id: str | None = None
+ # Whose turn this is and in which chat, sent on OpenRouter requests so
+ # Broadcast attributes each generation (see ``llm_attribution``).
+ user_id: str | None = None
+ session_id: str | None = None
# Live delivery channel drained concurrently by ``stream_chat_completion_baseline``
# so reasoning / text / tool events reach the SSE wire **during** the upstream
# LLM stream, not after ``_baseline_llm_caller`` returns. Before this was a
@@ -916,8 +921,16 @@ async def _baseline_llm_caller(
# which stricter local backends reject outright.
pass
elif is_openrouter_transport:
- # OR-only: Anthropic's compat endpoint 400s on usage.include + reasoning.
+ # OR-only: Anthropic's compat endpoint 400s on usage.include + reasoning
+ # and on the Broadcast fields, which only OpenRouter reads anyway.
extra_body.update(dict(_OPENROUTER_INCLUDE_USAGE_COST))
+ extra_body.update(
+ openrouter_attribution(
+ user_id=state.user_id,
+ session_id=state.session_id,
+ generation_name="copilot-baseline",
+ )
+ )
reasoning_param = reasoning_extra_body(
state.model, config.claude_agent_max_thinking_tokens
)
@@ -2393,6 +2406,8 @@ async def stream_chat_completion_baseline(
llm_credential_id=(
session.metadata.llm_credential_id if session is not None else None
),
+ user_id=user_id,
+ session_id=session_id,
)
# Queued, not yielded: the loop below yields them inside the try whose
diff --git a/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py b/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py
index eaac3e8348..b509950cb1 100644
--- a/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py
+++ b/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py
@@ -37,6 +37,8 @@ from backend.copilot.baseline.service import (
)
from backend.copilot.context import get_execution_context, set_execution_context
from backend.copilot.expert_context import ExpertSessionUnavailableError
+from backend.copilot.llm_attribution import set_turn_impersonated
+from backend.copilot.llm_attribution import settings as attribution_settings
from backend.copilot.model import ChatMessage, ChatSession
from backend.copilot.model_router import ResolvedModel
from backend.copilot.response_model import (
@@ -1208,9 +1210,80 @@ class TestBaselineCostExtraction:
create_mock.assert_awaited_once()
await_args = create_mock.await_args
assert await_args is not None
- assert await_args.kwargs["extra_body"] == {"usage": {"include": True}}
+ assert await_args.kwargs["extra_body"]["usage"] == {"include": True}
assert await_args.kwargs["stream_options"] == {"include_usage": True}
+ @pytest.mark.asyncio
+ async def test_openrouter_request_attributes_the_turn_for_broadcast(self):
+ """OpenRouter Broadcast puts the generation on the user's PostHog
+ person and the chat's ``$ai_session_id`` from these fields."""
+ state = _BaselineStreamState(
+ model="gpt-4o-mini", user_id="user-1", session_id="sess-1"
+ )
+ create_mock = AsyncMock(return_value=_make_stream_mock())
+ mock_client = MagicMock()
+ mock_client.chat.completions.create = create_mock
+
+ with (
+ patch(
+ "backend.copilot.baseline.service._get_main_client",
+ return_value=mock_client,
+ ),
+ patch("backend.copilot.baseline.service.config.use_openrouter", True),
+ patch("backend.copilot.baseline.service.config.api_key", "or-key"),
+ patch(
+ "backend.copilot.baseline.service.config.base_url",
+ "https://openrouter.ai/api/v1",
+ ),
+ ):
+ set_turn_impersonated(True)
+ try:
+ await _baseline_llm_caller(
+ messages=[{"role": "user", "content": "hi"}],
+ tools=[],
+ state=state,
+ )
+ finally:
+ set_turn_impersonated(False)
+
+ assert create_mock.await_args is not None
+ extra_body = create_mock.await_args.kwargs["extra_body"]
+ assert extra_body["user"] == "user-1"
+ assert extra_body["session_id"] == "sess-1"
+ assert extra_body["trace"] == {
+ "generation_name": "copilot-baseline",
+ "environment": attribution_settings.config.app_env.value,
+ "impersonated": True,
+ }
+
+ @pytest.mark.asyncio
+ async def test_direct_anthropic_request_carries_no_broadcast_fields(self):
+ """Anthropic's OpenAI-compat endpoint 400s on unknown body fields,
+ and Broadcast never sees a call that skips OpenRouter."""
+ state = _BaselineStreamState(
+ model="claude-sonnet-4-6", user_id="user-1", session_id="sess-1"
+ )
+ create_mock = AsyncMock(return_value=_make_stream_mock())
+ mock_client = MagicMock()
+ mock_client.chat.completions.create = create_mock
+
+ with (
+ patch(
+ "backend.copilot.baseline.service._get_main_client",
+ return_value=mock_client,
+ ),
+ patch("backend.copilot.baseline.service.config.use_openrouter", False),
+ ):
+ await _baseline_llm_caller(
+ messages=[{"role": "user", "content": "hi"}],
+ tools=[],
+ state=state,
+ )
+
+ assert create_mock.await_args is not None
+ extra_body = create_mock.await_args.kwargs["extra_body"] or {}
+ assert not {"user", "session_id", "trace"} & extra_body.keys()
+
class TestMidLoopPendingFlushOrdering:
"""Regression test for the mid-loop pending drain ordering invariant.
diff --git a/autogpt_platform/backend/backend/copilot/executor/processor.py b/autogpt_platform/backend/backend/copilot/executor/processor.py
index 1d01cd47ac..3ce2274e77 100644
--- a/autogpt_platform/backend/backend/copilot/executor/processor.py
+++ b/autogpt_platform/backend/backend/copilot/executor/processor.py
@@ -25,6 +25,7 @@ from backend.copilot.expert_context import (
EXPERT_SESSION_TEMPORARY_MESSAGE,
ExpertSessionUnavailableError,
)
+from backend.copilot.llm_attribution import set_turn_impersonated
from backend.copilot.response_model import StreamError, StreamStatus
from backend.copilot.sdk import service as sdk_service
from backend.copilot.sdk.dummy import stream_chat_completion_dummy
@@ -742,6 +743,7 @@ class CoPilotProcessor:
# same events as they are produced.
envelope = taint_for_source_platform(entry.envelope, session)
set_turn_unattended(session, scheduled=entry.unattended)
+ set_turn_impersonated(entry.impersonated)
set_turn_credential_pins(entry.credential_pins)
turn.engine_started(engine)
raw_stream = stream_fn(
diff --git a/autogpt_platform/backend/backend/copilot/executor/processor_test.py b/autogpt_platform/backend/backend/copilot/executor/processor_test.py
index dd2610123a..ff82c5a98d 100644
--- a/autogpt_platform/backend/backend/copilot/executor/processor_test.py
+++ b/autogpt_platform/backend/backend/copilot/executor/processor_test.py
@@ -33,6 +33,7 @@ from backend.copilot.expert_context import (
EXPERT_SESSION_TEMPORARY_MESSAGE,
ExpertSessionUnavailableError,
)
+from backend.copilot.llm_attribution import is_turn_impersonated
from backend.copilot.model import ChatSession
from backend.copilot.rate_limit import UserPaywalledError
from backend.copilot.response_model import (
@@ -218,6 +219,7 @@ class TestExecuteAsyncScheduledTurnContext:
def stream(**_kwargs):
seen["unattended"] = is_unattended_turn()
seen["pins"] = dict(turn_credential_pins())
+ seen["impersonated"] = is_turn_impersonated()
return MagicMock()
with (
@@ -260,13 +262,21 @@ class TestExecuteAsyncScheduledTurnContext:
seen = await self._seen_by_the_turn(entry)
- assert seen == {"unattended": True, "pins": {"exa": pin}}
+ assert seen == {"unattended": True, "pins": {"exa": pin}, "impersonated": False}
@pytest.mark.asyncio
async def test_a_typed_turn_is_watched_and_has_no_pins(self) -> None:
seen = await self._seen_by_the_turn(_make_entry())
- assert seen == {"unattended": False, "pins": {}}
+ assert seen == {"unattended": False, "pins": {}, "impersonated": False}
+
+ @pytest.mark.asyncio
+ async def test_an_impersonated_turn_is_marked_for_llm_analytics(self) -> None:
+ entry = _make_entry().model_copy(update={"impersonated": True})
+
+ seen = await self._seen_by_the_turn(entry)
+
+ assert seen["impersonated"] is True
class TestExecuteAsyncAclose:
diff --git a/autogpt_platform/backend/backend/copilot/executor/utils.py b/autogpt_platform/backend/backend/copilot/executor/utils.py
index 4cb63f67d2..b17beb0307 100644
--- a/autogpt_platform/backend/backend/copilot/executor/utils.py
+++ b/autogpt_platform/backend/backend/copilot/executor/utils.py
@@ -336,6 +336,11 @@ class CoPilotExecutionEntry(BaseModel):
is the user's own. Tools then never hand a question back to the user
(SECRT-2804); see ``set_turn_unattended``."""
+ impersonated: bool = False
+ """An admin impersonating the user sent the message that started this
+ turn. Marks the turn's LLM analytics rather than dropping them; see
+ ``llm_attribution.set_turn_impersonated``."""
+
credential_pins: CredentialPins = Field(default_factory=dict)
"""``{provider: pin}``: the accounts the schedule that fired this turn was
set up to run on. They win over the chat's own picks, and a pinned account
@@ -393,6 +398,7 @@ async def enqueue_copilot_turn(
*,
envelope: TurnEnvelope,
unattended: bool = False,
+ impersonated: bool = False,
credential_pins: CredentialPins | None = None,
scheduled: ScheduledTurnOrigin | None = None,
trace_headers: Mapping[str, str] | None = None,
@@ -443,6 +449,7 @@ async def enqueue_copilot_turn(
message_metadata=message_metadata,
envelope=envelope,
unattended=unattended,
+ impersonated=impersonated,
credential_pins=credential_pins or {},
scheduled=scheduled,
trace_headers=(
@@ -485,6 +492,7 @@ async def schedule_turn(
credential_pins: CredentialPins | None = None,
scheduled: ScheduledTurnOrigin | None = None,
delegated: bool = False,
+ impersonated: bool = False,
) -> None:
"""End-to-end "start a copilot turn": reserve a per-user concurrency
slot, register the session in the stream registry, then publish the
@@ -556,6 +564,7 @@ async def schedule_turn(
unattended=unattended,
credential_pins=credential_pins,
scheduled=scheduled,
+ impersonated=impersonated,
)
@@ -585,6 +594,7 @@ async def dispatch_turn(
scheduled: ScheduledTurnOrigin | None = None,
root: bool = False,
envelope: TurnEnvelope | None = None,
+ impersonated: bool = False,
) -> None:
"""Within an already-held turn slot, register the session in the
stream registry, publish the work to the executor queue, and
@@ -674,6 +684,7 @@ async def dispatch_turn(
message_metadata=message_metadata,
envelope=envelope,
unattended=unattended,
+ impersonated=impersonated,
credential_pins=credential_pins,
scheduled=scheduled,
trace_headers=trace_headers,
@@ -707,6 +718,7 @@ async def queue_spawned_turn(
permissions: CopilotPermissions | None,
spawn: SpawnRequest | None,
message_metadata: dict[str, Any] | None,
+ impersonated: bool = False,
) -> None:
"""Queue a turn another session started, to begin when a slot frees.
@@ -739,6 +751,7 @@ async def queue_spawned_turn(
tool_call_id=tool_call_id,
tool_name=tool_name,
only_if_idle=True,
+ impersonated=impersonated,
)
except BaseException:
await release_turn(envelope)
@@ -832,6 +845,7 @@ async def schedule_chat_turn(
llm_credential_id: str | None = None,
permissions: CopilotPermissions | None = None,
request_arrival_at: float = 0.0,
+ impersonated: bool = False,
) -> str | None:
"""HTTP-chat-flavoured :func:`schedule_turn`: acquires the user's
concurrency slot, persists the user message *inside* the slot, then
@@ -915,6 +929,7 @@ async def schedule_chat_turn(
llm_credential_id=llm_credential_id,
permissions=permissions,
request_arrival_at=request_arrival_at,
+ impersonated=impersonated,
)
return turn_id
diff --git a/autogpt_platform/backend/backend/copilot/executor/utils_test.py b/autogpt_platform/backend/backend/copilot/executor/utils_test.py
index c8a8be32f3..5226b63629 100644
--- a/autogpt_platform/backend/backend/copilot/executor/utils_test.py
+++ b/autogpt_platform/backend/backend/copilot/executor/utils_test.py
@@ -126,6 +126,17 @@ class TestCoPilotExecutionEntry:
)
assert legacy.unattended is False
+ def test_impersonated_survives_the_queue_and_defaults_to_false(self):
+ entry = CoPilotExecutionEntry(
+ session_id="s1", user_id="u1", message="hi", impersonated=True
+ )
+ restored = CoPilotExecutionEntry.model_validate_json(entry.model_dump_json())
+ assert restored.impersonated is True
+ legacy = CoPilotExecutionEntry.model_validate_json(
+ '{"session_id": "s1", "user_id": "u1", "message": "hi"}'
+ )
+ assert legacy.impersonated is False
+
def test_credential_pins_survive_the_queue_and_default_to_none(self):
pin = CredentialPin(id="exa-new", title="Work key")
entry = CoPilotExecutionEntry(
@@ -267,6 +278,29 @@ async def test_schedule_chat_turn_leaves_a_typed_message_alone() -> None:
assert append.await_args.args[1].content == "what did I run yesterday"
assert dispatch.await_args.kwargs["message"] == "what did I run yesterday"
+ assert dispatch.await_args.kwargs["impersonated"] is False
+
+
+async def test_schedule_chat_turn_hands_impersonation_to_the_turn() -> None:
+ slot = MagicMock(admitted=True)
+
+ @asynccontextmanager
+ async def acquire(*_args, **_kwargs):
+ yield slot
+
+ dispatch = AsyncMock()
+ with (
+ patch.object(utils, "acquire_turn_slot", new=acquire),
+ patch("backend.copilot.model.append_and_save_message", new=AsyncMock()),
+ patch("backend.copilot.tracking.track_user_message", new=MagicMock()),
+ patch.object(utils, "dispatch_turn", new=dispatch),
+ ):
+ await utils.schedule_chat_turn(
+ session_id="s1", user_id="u1", message="hi", impersonated=True
+ )
+
+ assert dispatch.await_args.kwargs["impersonated"] is True
@pytest.mark.asyncio
@@ -363,3 +397,97 @@ class TestNarrowPermissions:
assert merged is not None
assert merged.tools_exclude is False
assert merged.effective_allowed_tools(ALL_TOOL_NAMES) == {"read_workspace_file"}
+
+
+async def test_dispatch_turn_puts_impersonation_on_the_queued_entry() -> None:
+ queue = MagicMock(publish_message=AsyncMock())
+ with (
+ patch.object(
+ utils,
+ "_admitted_turn_envelope",
+ new=AsyncMock(return_value=root_envelope("t1")),
+ ),
+ patch("backend.copilot.stream_registry.create_session", new=AsyncMock()),
+ patch(
+ "backend.util.clients.get_async_copilot_queue",
+ new=AsyncMock(return_value=queue),
+ ),
+ ):
+ await utils.dispatch_turn(
+ MagicMock(),
+ session_id="s1",
+ user_id="u1",
+ turn_id="t1",
+ message="hi",
+ impersonated=True,
+ )
+
+ entry = CoPilotExecutionEntry.model_validate_json(
+ queue.publish_message.await_args.kwargs["message"]
+ )
+ assert entry.impersonated is True
+
+
+async def test_schedule_turn_hands_impersonation_to_the_turn() -> None:
+ slot = MagicMock(admitted=True)
+
+ @asynccontextmanager
+ async def acquire(*_args, **_kwargs):
+ yield slot
+
+ dispatch = AsyncMock()
+ with (
+ patch(
+ "backend.copilot.turn_queue.count_inflight_turns",
+ new=AsyncMock(return_value=0),
+ ),
+ patch.object(utils, "acquire_turn_slot", new=acquire),
+ patch.object(utils, "dispatch_turn", new=dispatch),
+ ):
+ await utils.schedule_turn(
+ session_id="s1", user_id="u1", turn_id="t1", message="hi"
+ )
+ assert dispatch.await_args.kwargs["impersonated"] is False
+
+ await utils.schedule_turn(
+ session_id="s1",
+ user_id="u1",
+ turn_id="t2",
+ message="hi",
+ impersonated=True,
+ )
+ assert dispatch.await_args.kwargs["impersonated"] is True
+
+
+async def test_queue_spawned_turn_stores_impersonation_with_the_queued_row() -> None:
+ enqueue = AsyncMock()
+ with (
+ patch(
+ "backend.copilot.turn_queue.count_inflight_turns",
+ new=AsyncMock(return_value=0),
+ ),
+ patch.object(
+ utils,
+ "_admitted_turn_envelope",
+ new=AsyncMock(return_value=root_envelope("t1")),
+ ),
+ patch("backend.copilot.turn_queue.enqueue_turn", new=enqueue),
+ ):
+ await utils.queue_spawned_turn(
+ session_id="s1",
+ user_id="u1",
+ message="hi",
+ tool_call_id="sub:parent",
+ tool_name="run_sub_session",
+ llm_auth_provider="platform",
+ llm_credential_id=None,
+ permissions=None,
+ spawn=None,
+ message_metadata=None,
+ impersonated=True,
+ )
+
+ assert enqueue.await_args.kwargs["impersonated"] is True
diff --git a/autogpt_platform/backend/backend/copilot/feedback.py b/autogpt_platform/backend/backend/copilot/feedback.py
index de565a89e3..10400ccd27 100644
--- a/autogpt_platform/backend/backend/copilot/feedback.py
+++ b/autogpt_platform/backend/backend/copilot/feedback.py
@@ -13,10 +13,20 @@ to Langfuse as a score named after the rating ("user-feedback": 1 = up,
The score reuses the rating's id, so rating a reply again updates its score
instead of adding one. A Langfuse failure is logged and shows in the
response; the rating itself is already saved.
+
+Each rating also goes to PostHog LLM analytics as an ``$ai_metric`` event on
+the user (distinct id = user id), for every user regardless of cookie
+consent, like the generations OpenRouter Broadcast sends there.
+``$ai_session_id`` is the chat session id, the same value the chat's
+``$ai_generation`` events carry, which is what joins a rating to the
+generations of its chat; ``message_id`` names the reply. It carries no
+``$ai_trace_id``: Broadcast assigns the generations' trace ids, so we have
+none to match. Rating again sends another event; the latest per
+``feedback_id`` is the rating that stands.
"""
import logging
-from typing import Literal
+from typing import Any, Literal
from langfuse import get_client
from langfuse.api import CreateScoreRequest, ScoreDataType
@@ -24,6 +34,8 @@ from prisma.models import ChatMessage as PrismaChatMessage
from pydantic import BaseModel, Field
from backend.copilot import feedback_db
+from backend.util.posthog_client import capture
+from backend.util.posthog_events import PostHogEvent
from backend.util.settings import Settings
logger = logging.getLogger(__name__)
@@ -70,11 +82,14 @@ async def record_message_feedback(
score_name: FeedbackScoreName,
score_value: FeedbackScoreValue,
comment: str | None,
+ impersonated: bool = False,
) -> MessageFeedbackResponse | None:
- """Save a rating of one reply in *user_id*'s chat and score it in Langfuse.
+ """Save a rating of one reply in *user_id*'s chat and score it in Langfuse
+ and PostHog.
*message_id* is the id the chat UI knows the reply by: ``<session>-seq-<N>``
for a reply loaded from the chat history, or the message row's own id.
+ *impersonated* marks a rating an admin gave while acting as the user.
Returns ``None`` when the session is not the caller's or *message_id* is
not a reply in it.
@@ -91,6 +106,16 @@ async def record_message_feedback(
comment=comment,
langfuse_trace_id=message.langfuseTraceId,
)
+ _send_to_posthog(
+ user_id=user_id,
+ feedback_id=feedback_id,
+ session_id=session_id,
+ message=message,
+ score_name=score_name,
+ score_value=score_value,
+ comment=comment,
+ impersonated=impersonated,
+ )
target = await _score_in_langfuse(
feedback_id=feedback_id,
session_id=session_id,
@@ -174,3 +199,31 @@ async def _score_in_langfuse(
)
return None
return "trace" if trace_id else "session"
+
+
+def _send_to_posthog(
+ *,
+ user_id: str,
+ feedback_id: str,
+ session_id: str,
+ message: PrismaChatMessage,
+ score_name: FeedbackScoreName,
+ score_value: FeedbackScoreValue,
+ comment: str | None,
+ impersonated: bool,
+) -> None:
+ """Queue the rating as an ``$ai_metric`` event. ``capture`` logs and
+ swallows a PostHog failure, so the rating is never lost to one."""
+ properties: dict[str, Any] = {
+ "$ai_metric_name": score_name,
+ "$ai_metric_value": score_value,
+ "$ai_session_id": session_id,
+ "chat_session_id": session_id,
+ "message_id": message.id,
+ "message_sequence": message.sequence,
+ "feedback_id": feedback_id,
+ "impersonated": impersonated,
+ }
+ if comment:
+ properties["$ai_feedback_text"] = comment
+ capture(user_id, PostHogEvent.AI_METRIC, properties)
diff --git a/autogpt_platform/backend/backend/copilot/feedback_test.py b/autogpt_platform/backend/backend/copilot/feedback_test.py
index 92ddc2d59e..556363b4f8 100644
--- a/autogpt_platform/backend/backend/copilot/feedback_test.py
+++ b/autogpt_platform/backend/backend/copilot/feedback_test.py
@@ -1,10 +1,161 @@
-"""Tests for reading the chat UI's message ids in ``copilot.feedback``."""
+"""Tests for ``copilot.feedback``: reading the chat UI's message ids, and
+sending a rating to PostHog LLM analytics.
+
+The route tests (``backend/api/features/chat/feedback_test.py``) cover the
+Postgres and Langfuse halves.
+"""
+
+from datetime import UTC, datetime
+from unittest.mock import AsyncMock, MagicMock
import pytest
+import pytest_mock
+from prisma.models import ChatMessage as PrismaChatMessage
-from backend.copilot.feedback import sequence_from_ui_message_id
+from backend.copilot import feedback, feedback_db
+from backend.copilot.feedback import (
+ record_message_feedback,
+ sequence_from_ui_message_id,
+)
+from backend.util import posthog_client
+from backend.util.posthog_events import PostHogEvent
SESSION_ID = "8d2f5c1e-2b7a-4c55-9d0e-3f6a1b2c4d5e"
+USER_ID = "3b1f0e2a-7c4d-4e5f-8a9b-0c1d2e3f4a5b"
+MESSAGE_ROW_ID = "0f9e8d7c-6b5a-4938-8271-605f4e3d2c1b"
+FEEDBACK_ID = "c3a1b2d4-e5f6-4789-8abc-def012345678"
+
+
+def reply(mocker: pytest_mock.MockerFixture) -> PrismaChatMessage:
+ message = PrismaChatMessage(
+ id=MESSAGE_ROW_ID,
+ createdAt=datetime.now(UTC),
+ sessionId=SESSION_ID,
+ role="assistant",
+ content="Here is the report.",
+ sequence=7,
+ langfuseTraceId="1edf31f11b1693cc6103f358c1481694",
+ )
+ mocker.patch.object(
+ feedback_db, "get_rateable_message", new=AsyncMock(return_value=message)
+ )
+ mocker.patch.object(
+ feedback_db,
+ "upsert_message_feedback",
+ new=AsyncMock(return_value=FEEDBACK_ID),
+ )
+ return message
+
+
+def langfuse_score(mocker: pytest_mock.MockerFixture) -> AsyncMock:
+ secrets = feedback.settings.secrets
+ mocker.patch.object(secrets, "langfuse_public_key", "pk-lf-test")
+ mocker.patch.object(secrets, "langfuse_secret_key", "sk-lf-test")
+ langfuse = MagicMock()
+ langfuse.async_api.score.create = AsyncMock()
+ mocker.patch.object(feedback, "get_client", return_value=langfuse)
+ return langfuse.async_api.score.create
+
+
+async def _rate(**overrides):
+ kwargs = {
+ "user_id": USER_ID,
+ "session_id": SESSION_ID,
+ "message_id": f"{SESSION_ID}-seq-7",
+ "score_name": "user-feedback",
+ "score_value": 0,
+ "comment": "It edited the wrong file",
+ **overrides,
+ }
+ return await record_message_feedback(**kwargs)
+
+
+async def test_thumbs_down_is_an_ai_metric_on_the_user_and_chat(
+ mocker: pytest_mock.MockerFixture, langfuse_score: AsyncMock
+) -> None:
+ capture = mocker.patch.object(feedback, "capture")
+
+ response = await _rate()
+
+ assert response is not None
+ capture.assert_called_once_with(
+ USER_ID,
+ PostHogEvent.AI_METRIC,
+ {
+ "$ai_metric_name": "user-feedback",
+ "$ai_metric_value": 0,
+ "$ai_session_id": SESSION_ID,
+ "chat_session_id": SESSION_ID,
+ "message_id": MESSAGE_ROW_ID,
+ "message_sequence": 7,
+ "feedback_id": FEEDBACK_ID,
+ "impersonated": False,
+ "$ai_feedback_text": "It edited the wrong file",
+ },
+ )
+ langfuse_score.assert_awaited_once()
+ assert response.langfuse_target == "trace"
+
+
[email protected]("reply", "langfuse_score")
+async def test_copy_carries_no_feedback_text(
+ mocker: pytest_mock.MockerFixture,
+) -> None:
+ capture = mocker.patch.object(feedback, "capture")
+
+ await _rate(score_name="copy", score_value=1, comment=None)
+
+ properties = capture.call_args.args[2]
+ assert properties["$ai_metric_name"] == "copy"
+ assert properties["$ai_metric_value"] == 1
+ assert "$ai_feedback_text" not in properties
+
+
[email protected]("reply", "langfuse_score")
+async def test_impersonated_rating_is_marked(
+ mocker: pytest_mock.MockerFixture,
+) -> None:
+ capture = mocker.patch.object(feedback, "capture")
+
+ await _rate(impersonated=True)
+
+ assert capture.call_args.args[2]["impersonated"] is True
+
+
+async def test_posthog_failure_keeps_the_rating_and_the_langfuse_score(
+ mocker: pytest_mock.MockerFixture, langfuse_score: AsyncMock
+) -> None:
+ """The real ``capture`` runs against a client that raises."""
+ client = MagicMock()
+ client.capture.side_effect = RuntimeError("posthog is down")
+ mocker.patch.object(posthog_client, "get_posthog_client", return_value=client)
+
+ response = await _rate()
+
+ client.capture.assert_called_once()
+ assert client.capture.call_args.kwargs["distinct_id"] == USER_ID
+ assert response is not None
+ assert response.id == FEEDBACK_ID
+ langfuse_score.assert_awaited_once()
+
+
+async def test_unknown_reply_sends_nothing(mocker: pytest_mock.MockerFixture) -> None:
+ mocker.patch.object(
+ feedback_db, "get_rateable_message", new=AsyncMock(return_value=None)
+ )
+ capture = mocker.patch.object(feedback, "capture")
+
+ assert await _rate() is None
+ capture.assert_not_called()
@pytest.mark.parametrize(
diff --git a/autogpt_platform/backend/backend/copilot/llm_attribution.py b/autogpt_platform/backend/backend/copilot/llm_attribution.py
new file mode 100644
index 0000000000..1f555493e8
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/llm_attribution.py
@@ -0,0 +1,81 @@
+"""Who an AutoPilot LLM call is for, in the fields OpenRouter Broadcast reads.
+
+OpenRouter Broadcast sends every generation to PostHog LLM analytics as an
+``$ai_generation`` event (and to Langfuse). It maps these request body fields
+(https://openrouter.ai/docs/guides/features/broadcast/posthog):
+
+- ``user`` becomes the event's ``distinct_id``. It is the platform user id,
+ the distinct id every other backend and browser event uses, so the
+ generation lands on the same person as their product events and replays.
+- ``session_id`` becomes ``$ai_session_id``: the chat session id.
+- ``trace.generation_name`` becomes ``$ai_span_name``; every other ``trace``
+ key arrives as ``metadata_<key>`` (``metadata_environment``,
+ ``metadata_impersonated``).
+
+Only calls that go through OpenRouter are covered. Direct Anthropic, Claude
+Code subscription and user-connected (Codex) routes never pass OpenRouter, so
+Broadcast sees none of their generations.
+"""
+
+from contextvars import ContextVar
+from typing import Any, Literal
+
+from autogpt_libs.auth.dependencies import IMPERSONATION_HEADER_NAME
+from fastapi import Request
+
+from backend.util.settings import Settings
+
+settings = Settings()
+
+# OpenRouter's limit on ``user``. ``session_id`` allows 256; chat session ids
+# are UUIDs, so the shorter cap never cuts one.
+_MAX_ID_CHARS = 128
+
+GenerationName = Literal["copilot-title", "copilot-sdk", "copilot-baseline"]
+
+# Whether an admin impersonating the user started the running turn. The
+# executor sets it for every turn, so a value never carries over.
+_turn_impersonated: ContextVar[bool] = ContextVar("_turn_impersonated", default=False)
+
+
+def openrouter_attribution(
+ *,
+ user_id: str | None,
+ session_id: str | None,
+ generation_name: GenerationName,
+) -> dict[str, Any]:
+ """The body fields that attribute one OpenRouter request in Broadcast."""
+ body: dict[str, Any] = {
+ "trace": {
+ "generation_name": generation_name,
+ "environment": _environment(),
+ "impersonated": is_turn_impersonated(),
+ }
+ }
+ if user_id:
+ body["user"] = user_id[:_MAX_ID_CHARS]
+ if session_id:
+ body["session_id"] = session_id[:_MAX_ID_CHARS]
+ return body
+
+
+def set_turn_impersonated(impersonated: bool) -> None:
+ """Mark the running turn as started by an impersonating admin, or not."""
+ _turn_impersonated.set(impersonated)
+
+
+def is_turn_impersonated() -> bool:
+ return _turn_impersonated.get()
+
+
+def is_impersonated_request(request: Request) -> bool:
+ """Whether an admin made *request* as another user.
+
+ The auth dependencies have already refused the header from anyone but an
+ admin by the time a route runs, so its presence is enough.
+ """
+ return bool(request.headers.get(IMPERSONATION_HEADER_NAME, "").strip())
+
+
+def _environment() -> str:
+ return settings.config.app_env.value
diff --git a/autogpt_platform/backend/backend/copilot/llm_attribution_test.py b/autogpt_platform/backend/backend/copilot/llm_attribution_test.py
new file mode 100644
index 0000000000..02964b6cd6
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/llm_attribution_test.py
@@ -0,0 +1,90 @@
+"""Tests for the OpenRouter Broadcast attribution fields."""
+
+import asyncio
+from unittest.mock import MagicMock
+
+import pytest
+
+from backend.copilot import llm_attribution
+from backend.copilot.llm_attribution import (
+ is_impersonated_request,
+ is_turn_impersonated,
+ openrouter_attribution,
+ set_turn_impersonated,
+)
+from backend.util.settings import AppEnvironment
+
+
[email protected](autouse=True)
+def _not_impersonated():
+ set_turn_impersonated(False)
+ yield
+ set_turn_impersonated(False)
+
+
+def test_attribution_names_the_user_chat_and_environment(mocker):
+ mocker.patch.object(
+ llm_attribution.settings.config, "app_env", AppEnvironment.DEVELOPMENT
+ )
+
+ body = openrouter_attribution(
+ user_id="user-1", session_id="sess-1", generation_name="copilot-title"
+ )
+
+ assert body == {
+ "user": "user-1",
+ "session_id": "sess-1",
+ "trace": {
+ "generation_name": "copilot-title",
+ "environment": "dev",
+ "impersonated": False,
+ },
+ }
+
+
+def test_attribution_without_a_user_or_chat_still_tags_the_environment():
+ body = openrouter_attribution(
+ user_id=None, session_id=None, generation_name="copilot-baseline"
+ )
+
+ assert set(body) == {"trace"}
+ assert body["trace"]["generation_name"] == "copilot-baseline"
+
+
+def test_attribution_marks_an_impersonated_turn():
+ set_turn_impersonated(True)
+
+ body = openrouter_attribution(
+ user_id="user-1", session_id="sess-1", generation_name="copilot-sdk"
+ )
+
+ assert body["trace"]["impersonated"] is True
+
+
+async def test_impersonation_does_not_leak_into_another_turn():
+ async def turn(impersonated: bool) -> bool:
+ set_turn_impersonated(impersonated)
+ await asyncio.sleep(0)
+ return is_turn_impersonated()
+
+ admin, user = await asyncio.gather(
+ asyncio.create_task(turn(True)), asyncio.create_task(turn(False))
+ )
+
+ assert (admin, user) == (True, False)
+
+
+ ("headers", "expected"),
+ [
+ ({"X-Act-As-User-Id": "target-user"}, True),
+ ({"X-Act-As-User-Id": " "}, False),
+ ({}, False),
+ ],
+)
+def test_is_impersonated_request(headers: dict[str, str], expected: bool):
+ request = MagicMock()
+ request.headers = headers
+
+ assert is_impersonated_request(request) is expected
diff --git a/autogpt_platform/backend/backend/copilot/sdk/env.py b/autogpt_platform/backend/backend/copilot/sdk/env.py
index 2a5979cc30..bed12de8aa 100644
--- a/autogpt_platform/backend/backend/copilot/sdk/env.py
+++ b/autogpt_platform/backend/backend/copilot/sdk/env.py
@@ -8,11 +8,13 @@ circular import through ``executor`` → ``credit`` → ``block_cost_config``).
from __future__ import annotations
+import json
import os
import re
from urllib.parse import urlparse
from backend.copilot.config import ChatConfig
+from backend.copilot.llm_attribution import openrouter_attribution
from backend.copilot.moonshot import is_moonshot_model, moonshot_context_window
from backend.copilot.sdk.subscription import validate_subscription
@@ -56,7 +58,10 @@ def build_sdk_env(
3. **Direct Anthropic** — subprocess inherits ``ANTHROPIC_API_KEY``
from the parent environment (no overrides needed).
4. **OpenRouter** (default) — overrides base URL and auth token to
- route through the proxy, with Langfuse trace headers.
+ route through the proxy, with the headers and body fields OpenRouter
+ Broadcast attributes generations by (user, chat session,
+ environment). Only this mode reaches Broadcast, so only its
+ generations show in PostHog LLM analytics and Langfuse broadcast.
All modes receive workspace isolation (``CLAUDE_CODE_TMPDIR``) and
security hardening env vars to prevent .claude.md loading, prompt
@@ -142,7 +147,9 @@ def build_sdk_env(
"CLAUDE_CODE_REFRESH_TOKEN": "", # prevent token refresh via subscription
}
- # Inject broadcast headers so OpenRouter forwards traces to Langfuse.
+ # Inject broadcast headers: OpenRouter Broadcast skips the
+ # Anthropic-native ``/messages`` endpoint unless ``x-session-id``
+ # is present.
def _safe(v: str) -> str:
return _HEADER_SAFE_RE.sub("", v).strip()[:_MAX_HEADER_VALUE_LEN]
@@ -153,6 +160,7 @@ def build_sdk_env(
parts.append(f"x-user-id: {_safe(user_id)}")
if parts:
env["ANTHROPIC_CUSTOM_HEADERS"] = "\n".join(parts)
+ env["CLAUDE_CODE_EXTRA_BODY"] = _broadcast_extra_body(session_id, user_id)
# --- Common: workspace isolation + security hardening (all modes) ---
# Route subagent temp files into the per-session workspace so output
@@ -258,3 +266,20 @@ def _autocompact_pct_for_model(model: str | None) -> int:
if model and "claude-sonnet-5" in model:
pct = min(round(pct * _SONNET_5_TOKENIZER_INFLATION), 90)
return pct
+
+
+def _broadcast_extra_body(session_id: str | None, user_id: str | None) -> str:
+ """``CLAUDE_CODE_EXTRA_BODY``: JSON the CLI merges into every request body
+ (main turn, sub-agents, compaction), carrying the fields OpenRouter
+ Broadcast attributes a generation by.
+
+ ``metadata.user_id`` replaces the CLI's own (a JSON blob of its device and
+ session ids) so that every user field on the request names the same
+ platform user.
+ """
+ body = openrouter_attribution(
+ user_id=user_id, session_id=session_id, generation_name="copilot-sdk"
+ )
+ if user_id:
+ body["metadata"] = {"user_id": body["user"]}
+ return json.dumps(body)
diff --git a/autogpt_platform/backend/backend/copilot/sdk/env_broadcast_test.py b/autogpt_platform/backend/backend/copilot/sdk/env_broadcast_test.py
new file mode 100644
index 0000000000..7094f4cf6a
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/sdk/env_broadcast_test.py
@@ -0,0 +1,106 @@
+"""The OpenRouter Broadcast fields ``build_sdk_env`` hands the Claude CLI.
+
+The CLI owns the request body, so the fields that attribute a generation in
+PostHog LLM analytics travel in ``CLAUDE_CODE_EXTRA_BODY``, which the CLI
+merges into every request it makes.
+"""
+
+import json
+from unittest.mock import patch
+
+import pytest
+
+from backend.copilot.config import ChatConfig
+from backend.copilot.llm_attribution import set_turn_impersonated
+from backend.copilot.llm_attribution import settings as attribution_settings
+from backend.copilot.sdk.env import build_sdk_env
+
+
+def _config(**overrides) -> ChatConfig:
+ fields = {
+ "use_claude_code_subscription": False,
+ "use_openrouter": True,
+ "api_key": "sk-or-test-key",
+ "base_url": "https://openrouter.ai/api/v1",
+ "thinking_standard_model": "anthropic/claude-sonnet-4-6",
+ "thinking_advanced_model": "anthropic/claude-opus-4-7",
+ "aux_api_key": "or-aux-key",
+ }
+ fields.update(overrides)
+ return ChatConfig(**fields)
+
+
[email protected](autouse=True)
+def _not_impersonated():
+ set_turn_impersonated(False)
+ yield
+ set_turn_impersonated(False)
+
+
+def _extra_body(**kwargs) -> dict:
+ with patch("backend.copilot.sdk.env.config", _config()):
+ env = build_sdk_env(**kwargs)
+ return json.loads(env["CLAUDE_CODE_EXTRA_BODY"])
+
+
+def test_openrouter_mode_attributes_every_request_to_the_user_and_chat():
+ body = _extra_body(session_id="sess-1", user_id="user-1")
+
+ assert body == {
+ "user": "user-1",
+ "session_id": "sess-1",
+ "metadata": {"user_id": "user-1"},
+ "trace": {
+ "generation_name": "copilot-sdk",
+ "environment": attribution_settings.config.app_env.value,
+ "impersonated": False,
+ },
+ }
+
+
+def test_openrouter_mode_keeps_the_session_header_broadcast_needs():
+ with patch("backend.copilot.sdk.env.config", _config()):
+ env = build_sdk_env(session_id="sess-1", user_id="user-1")
+
+ assert "x-session-id: sess-1" in env["ANTHROPIC_CUSTOM_HEADERS"]
+
+
+def test_an_impersonated_turn_is_marked():
+ set_turn_impersonated(True)
+
+ body = _extra_body(session_id="sess-1", user_id="user-1")
+
+ assert body["trace"]["impersonated"] is True
+
+
+def test_no_user_sends_no_user_fields():
+ body = _extra_body(session_id="sess-1")
+
+ assert "user" not in body
+ assert "metadata" not in body
+ assert body["session_id"] == "sess-1"
+
+
+def test_long_ids_are_cut_to_openrouters_limit():
+ body = _extra_body(session_id="s" * 300, user_id="u" * 300)
+
+ assert body["user"] == "u" * 128
+ assert body["metadata"] == {"user_id": "u" * 128}
+ assert body["session_id"] == "s" * 128
+
+
+def test_direct_anthropic_mode_sends_no_broadcast_fields():
+ """Broadcast never sees a call that skips OpenRouter."""
+ with patch("backend.copilot.sdk.env.config", _config(use_openrouter=False)):
+ env = build_sdk_env(session_id="sess-1", user_id="user-1")
+
+ assert "CLAUDE_CODE_EXTRA_BODY" not in env
+
+
+@patch("backend.copilot.sdk.env.validate_subscription")
+def test_subscription_mode_sends_no_broadcast_fields(_validate):
+ cfg = _config(use_claude_code_subscription=True)
+ with patch("backend.copilot.sdk.env.config", cfg):
+ env = build_sdk_env(session_id="sess-1", user_id="user-1")
+
+ assert "CLAUDE_CODE_EXTRA_BODY" not in env
diff --git a/autogpt_platform/backend/backend/copilot/sdk/session_waiter.py b/autogpt_platform/backend/backend/copilot/sdk/session_waiter.py
index b1251f8b10..621ffb61c4 100644
--- a/autogpt_platform/backend/backend/copilot/sdk/session_waiter.py
+++ b/autogpt_platform/backend/backend/copilot/sdk/session_waiter.py
@@ -28,6 +28,7 @@ from backend.copilot import stream_registry
from backend.copilot.active_turns import ConcurrentTurnLimitError
from backend.copilot.config import CopilotLlmAuthProvider
from backend.copilot.executor.utils import queue_spawned_turn, schedule_turn
+from backend.copilot.llm_attribution import is_turn_impersonated
from backend.copilot.model import (
CHAT_STATUS_QUEUED,
_get_session_lock,
@@ -278,6 +279,8 @@ async def run_copilot_turn_via_queue(
spawn=spawn,
message_metadata=message_metadata,
delegated=delegated,
+ # A child of an impersonated turn is the admin's work too.
+ impersonated=is_turn_impersonated(),
)
except TreeRefusal as refused:
return "refused", SessionResult(refusal=refused.message)
@@ -369,6 +372,7 @@ async def _queue_for_slot(
permissions=permissions,
spawn=spawn,
message_metadata=message_metadata,
+ impersonated=is_turn_impersonated(),
)
except TreeRefusal as refused:
return "refused", SessionResult(refusal=refused.message)
diff --git a/autogpt_platform/backend/backend/copilot/sdk/session_waiter_test.py b/autogpt_platform/backend/backend/copilot/sdk/session_waiter_test.py
index d1018bb798..26c67b839b 100644
--- a/autogpt_platform/backend/backend/copilot/sdk/session_waiter_test.py
+++ b/autogpt_platform/backend/backend/copilot/sdk/session_waiter_test.py
@@ -446,3 +446,77 @@ def _status_db(status: str) -> MagicMock:
db = MagicMock()
db.get_chat_session_status = AsyncMock(return_value=status)
return db
+
+
[email protected]("impersonated", [False, True])
+async def test_a_child_turn_carries_its_parents_impersonation(impersonated: bool):
+ """A sub-session started inside an impersonated turn is the admin's work
+ too, so its LLM analytics are marked the same way."""
+ scheduled = AsyncMock()
+
+ with (
+ patch(
+ "backend.copilot.sdk.session_waiter.is_turn_in_flight",
+ new=AsyncMock(return_value=False),
+ ),
+ patch(
+ "backend.copilot.sdk.session_waiter.is_turn_impersonated",
+ return_value=impersonated,
+ ),
+ patch("backend.copilot.sdk.session_waiter.schedule_turn", new=scheduled),
+ patch(
+ "backend.copilot.sdk.session_waiter.wait_for_session_result",
+ new=AsyncMock(return_value=("completed", SessionResult())),
+ ),
+ ):
+ await run_copilot_turn_via_queue(
+ session_id="sess-sub",
+ user_id="u1",
+ message="spawn",
+ timeout=5,
+ tool_call_id="sub:parent",
+ tool_name="run_sub_session",
+ )
+
+ assert scheduled.await_args.kwargs["impersonated"] is impersonated
+
+
+async def test_a_queued_child_turn_carries_its_parents_impersonation(
+ mock_session_lookup,
+):
+ """A delegated child the running cap sends to the queue keeps the flag
+ for when it is promoted."""
+ mock_session_lookup.return_value.metadata.delegated_by_session_id = "parent"
+ queued = AsyncMock()
+
+ with (
+ patch(
+ "backend.copilot.sdk.session_waiter.is_turn_in_flight",
+ new=AsyncMock(return_value=False),
+ ),
+ patch(
+ "backend.copilot.sdk.session_waiter.is_turn_impersonated",
+ return_value=True,
+ ),
+ patch(
+ "backend.copilot.sdk.session_waiter.schedule_turn",
+ new=AsyncMock(side_effect=active_turns.ConcurrentTurnLimitError()),
+ ),
+ patch("backend.copilot.sdk.session_waiter.queue_spawned_turn", new=queued),
+ patch(
+ "backend.copilot.sdk.session_waiter.wait_for_queued_session",
+ new=AsyncMock(return_value=("completed", SessionResult())),
+ ),
+ ):
+ await run_copilot_turn_via_queue(
+ session_id="sess-sub",
+ user_id="u1",
+ message="spawn",
+ timeout=5,
+ tool_call_id="sub:parent",
+ tool_name="run_sub_session",
+ )
+
+ assert queued.await_args.kwargs["impersonated"] is True
diff --git a/autogpt_platform/backend/backend/copilot/service.py b/autogpt_platform/backend/backend/copilot/service.py
index c7d90266af..9710de55a2 100644
--- a/autogpt_platform/backend/backend/copilot/service.py
+++ b/autogpt_platform/backend/backend/copilot/service.py
@@ -36,6 +36,7 @@ from .config import ChatConfig, CopilotLLMModel
from .expert_context import OWNED_BLOCK_TAGS as _EXPERT_BLOCK_TAGS
from .expert_context import build_expert_context, escape_prompt_xml_tags
from .expert_kickoff import is_expert_kickoff_message
+from .llm_attribution import openrouter_attribution
from .model import (
ChatMessage,
ChatSessionInfo,
@@ -827,24 +828,24 @@ async def _generate_session_title(
``create`` did not bill, so there is no cost to record.
"""
try:
- # Build extra_body for OpenRouter tracing and PostHog analytics.
+ # Build extra_body for OpenRouter cost and Broadcast attribution.
# ``usage: {"include": True}`` asks OR to embed the real billed
# cost into the final usage chunk — matches the baseline path's
# ``_OPENROUTER_INCLUDE_USAGE_COST`` pattern, same read path.
# Gated on the aux transport because Anthropic's OpenAI-compat
# endpoint (and any non-OR endpoint) rejects unknown extra_body
# fields with a 400 — the same gate the baseline path applies.
+ # Off OpenRouter, Broadcast never sees the call.
extra_body: dict[str, Any] = {}
if config.aux_uses_openrouter:
extra_body["usage"] = {"include": True}
- if user_id:
- extra_body["user"] = user_id[:128] # OpenRouter limit
- extra_body["posthogDistinctId"] = user_id
- if session_id:
- extra_body["session_id"] = session_id[:128] # OpenRouter limit
- extra_body["posthogProperties"] = {
- "environment": settings.config.app_env.value,
- }
+ extra_body.update(
+ openrouter_attribution(
+ user_id=user_id,
+ session_id=session_id,
+ generation_name="copilot-title",
+ )
+ )
# Normalize the title model for the aux client's transport: OR
# routes by full ``vendor/model`` slug, but Anthropic's
diff --git a/autogpt_platform/backend/backend/copilot/service_unit_test.py b/autogpt_platform/backend/backend/copilot/service_unit_test.py
index 23e3877987..e9b66293ef 100644
--- a/autogpt_platform/backend/backend/copilot/service_unit_test.py
+++ b/autogpt_platform/backend/backend/copilot/service_unit_test.py
@@ -28,6 +28,7 @@ from backend.copilot.service import (
_record_title_generation_cost,
_title_usage_from_response,
_update_title_async,
+ settings,
)
from backend.util.settings import Config
@@ -667,6 +668,36 @@ class TestGenerateSessionTitle:
assert extra_body["usage"] == {"include": True}
assert extra_body["user"] == "user-abc"
assert extra_body["session_id"] == "sess-abc"
+ assert extra_body["trace"] == {
+ "generation_name": "copilot-title",
+ "environment": settings.config.app_env.value,
+ "impersonated": False,
+ }
+
+ @pytest.mark.asyncio
+ async def test_title_call_off_openrouter_carries_no_broadcast_fields(self):
+ """Broadcast only sees OpenRouter calls, and a direct-Anthropic aux
+ endpoint 400s on unknown body fields."""
+ resp = _build_completion(content="Title")
+ helper = AsyncMock(return_value=resp)
+ with (
+ patch(
+ "backend.copilot.service.call_provider_openai_compat_sync",
+ new=helper,
+ ),
+ patch(
+ "backend.copilot.service.config",
+ MagicMock(
+ aux_uses_openrouter=False,
+ aux_provider_label="anthropic",
+ title_model="anthropic/claude-haiku-4.5",
+ ),
+ ),
+ ):
+ await _generate_session_title(
+ "hello", user_id="user-abc", session_id="sess-abc"
+ )
+ assert helper.await_args.kwargs["extra_body"] is None
@pytest.mark.asyncio
async def test_title_model_normalized_when_aux_is_anthropic(self):
diff --git a/autogpt_platform/backend/backend/copilot/trial_reads_test.py b/autogpt_platform/backend/backend/copilot/trial_reads_test.py
index 9f0608dec5..aac2fdacea 100644
--- a/autogpt_platform/backend/backend/copilot/trial_reads_test.py
+++ b/autogpt_platform/backend/backend/copilot/trial_reads_test.py
@@ -152,7 +152,11 @@ async def test_the_stream_route_runs_its_checks_in_one_block():
pytest.raises(_StopTheTurn),
):
await stream_chat_post(
- "session-1", StreamChatRequest(message="hi"), "u1", MagicMock()
+ "session-1",
+ StreamChatRequest(message="hi"),
+ "u1",
+ MagicMock(),
+ http_request=MagicMock(),
)
assert read.await_count == 1
diff --git a/autogpt_platform/backend/backend/copilot/turn_queue.py b/autogpt_platform/backend/backend/copilot/turn_queue.py
index 48768c3987..11e8770d66 100644
--- a/autogpt_platform/backend/backend/copilot/turn_queue.py
+++ b/autogpt_platform/backend/backend/copilot/turn_queue.py
@@ -10,7 +10,8 @@ text column:
The user's pending message itself is just a normal ChatMessage row (no
status of its own). The dispatcher's submit-time payload (``file_ids``,
-``model``, ``permissions``, ``context``, ``request_arrival_at``)
+``model``, ``permissions``, ``context``, ``request_arrival_at``,
+``impersonated``)
is stashed in that row's ``metadata`` JSONB so a later promotion can
replay the turn faithfully.
@@ -148,6 +149,7 @@ async def try_enqueue_turn(
permissions: Mapping[str, Any] | None = None,
request_arrival_at: float = 0.0,
envelope: TurnEnvelope | None = None,
+ impersonated: bool = False,
) -> ChatMessage | None:
"""Admit a queued turn against the user's hard cap.
@@ -174,6 +176,7 @@ async def try_enqueue_turn(
permissions=permissions,
request_arrival_at=request_arrival_at,
envelope=envelope,
+ impersonated=impersonated,
)
@@ -196,6 +199,7 @@ async def enqueue_turn(
tool_call_id: str | None = None,
tool_name: str | None = None,
only_if_idle: bool = False,
+ impersonated: bool = False,
) -> ChatMessage | None:
"""Persist the user's pending message and flip the session to
``"queued"``. Caller is responsible for the in-flight cap check
@@ -220,6 +224,8 @@ async def enqueue_turn(
metadata["permissions"] = dict(permissions)
if request_arrival_at:
metadata["request_arrival_at"] = request_arrival_at
+ if impersonated:
+ metadata["impersonated"] = True
# A wake starts under the envelope its call was held under, and a spawned
# child under the one admitted for it, whichever turn frees the slot.
if envelope is not None:
@@ -514,6 +520,7 @@ async def _promote_head(user_id: str) -> bool | None:
llm_credential_id=head.metadata.llm_credential_id,
permissions=_promotion_permissions(head, metadata, spawned=spawned),
request_arrival_at=float(metadata.get("request_arrival_at") or 0.0),
+ impersonated=metadata.get("impersonated") is True,
tool_call_id=metadata.get("tool_call_id") or "chat_stream",
tool_name=metadata.get("tool_name") or "chat",
# A typed message is a root, as the chat route makes it, not the
diff --git a/autogpt_platform/backend/backend/copilot/turn_queue_test.py b/autogpt_platform/backend/backend/copilot/turn_queue_test.py
index d7b46b3530..b9a5014d41 100644
--- a/autogpt_platform/backend/backend/copilot/turn_queue_test.py
+++ b/autogpt_platform/backend/backend/copilot/turn_queue_test.py
@@ -135,6 +135,27 @@ async def test_enqueue_turn_packs_metadata_into_metadata_payload() -> None:
)
+async def test_enqueue_turn_records_an_impersonated_turn() -> None:
+ db = MagicMock()
+ db.get_next_sequence = AsyncMock(return_value=1)
+ db.add_chat_message = AsyncMock(return_value=_pyd_message())
+ db.update_chat_session_status = AsyncMock(return_value=True)
+ with (
+ patch.object(turn_queue, "chat_db", return_value=db),
+ patch.object(turn_queue, "count_inflight_turns", new=AsyncMock(return_value=0)),
+ patch.object(turn_queue, "invalidate_session_cache", new=AsyncMock()),
+ ):
+ await turn_queue.try_enqueue_turn(
+ user_id="u1",
+ inflight_cap=15,
+ session_id="s1",
+ message="hello",
+ impersonated=True,
+ )
+ assert db.add_chat_message.call_args.kwargs["metadata"]["impersonated"] is True
+
+
@pytest.mark.asyncio
async def test_enqueue_turn_only_includes_default_transport_without_extra_params() -> (
None
@@ -1441,11 +1462,16 @@ async def test_dispatch_defers_on_rate_limit_unavailable() -> None:
@pytest.mark.asyncio
-async def test_dispatch_happy_path_claims_and_dispatches() -> None:
[email protected]("impersonated", [False, True])
+async def test_dispatch_happy_path_claims_and_dispatches(impersonated: bool) -> None:
"""All gates pass → claim session queued → running, build a TurnSlot,
- dispatch_turn, invalidate cache, return True."""
+ dispatch_turn, invalidate cache, return True. A turn queued by an
+ impersonating admin is promoted as one."""
head = _queued_row(session_id="s1")
- pending = _pyd_message(metadata={"mode": "extended_thinking"})
+ stored: dict = {"mode": "extended_thinking"}
+ if impersonated:
+ stored["impersonated"] = True
+ pending = _pyd_message(metadata=stored)
db = MagicMock()
db.admit_chat_session_turn = AsyncMock(return_value="admitted")
db.update_chat_session_status = AsyncMock(return_value=True)
@@ -1476,6 +1502,7 @@ async def test_dispatch_happy_path_claims_and_dispatches() -> None:
promoted = await turn_queue.dispatch_next_for_user("u1")
assert promoted is True
dispatch_turn_mock.assert_awaited_once()
+ assert dispatch_turn_mock.await_args.kwargs["impersonated"] is impersonated
invalidate.assert_awaited_once_with("s1")
# Claimed queued → running under the user's cap; nothing restored.
assert db.admit_chat_session_turn.await_args.kwargs == {
diff --git a/autogpt_platform/backend/backend/util/posthog_events.py b/autogpt_platform/backend/backend/util/posthog_events.py
index 01ab56099c..f550ebe770 100644
--- a/autogpt_platform/backend/backend/util/posthog_events.py
+++ b/autogpt_platform/backend/backend/util/posthog_events.py
@@ -68,6 +68,11 @@ class PostHogEvent(StrEnum):
# event, not a user action; keep it out of funnels.
SET_PERSON_PROPERTIES = "$set"
+ # LLM analytics (SECRT-2730): backend/copilot/feedback.py. PostHog's own
+ # event for a score on a generative AI trace, named by PostHog rather
+ # than the analytics plan; a thumbs up/down or copy of an AutoPilot reply.
+ AI_METRIC = "$ai_metric"
+
# Key moments (SECRT-2723): backend/util/product_analytics.py
SIGNUP_COMPLETED = "signup_completed"
ONBOARDING_COMPLETED = "onboarding_completed"
diff --git a/autogpt_platform/backend/backend/util/posthog_events_test.py b/autogpt_platform/backend/backend/util/posthog_events_test.py
index 1cca68c21c..5e6e533713 100644
--- a/autogpt_platform/backend/backend/util/posthog_events_test.py
+++ b/autogpt_platform/backend/backend/util/posthog_events_test.py
@@ -41,6 +41,7 @@ LIVE_EVENT_NAMES = {
"trial_converted",
"payment_failed",
"$set",
+ "$ai_metric",
"signup_completed",
"onboarding_completed",
"checkout_started",
diff --git a/autogpt_platform/frontend/src/app/api/openapi.json b/autogpt_platform/frontend/src/app/api/openapi.json
index 4fd7d69f7c..b28802a1bc 100644
--- a/autogpt_platform/frontend/src/app/api/openapi.json
+++ b/autogpt_platform/frontend/src/app/api/openapi.json
@@ -4428,7 +4428,7 @@
"post": {
"tags": ["v2", "chat", "chat"],
"summary": "Submit message feedback",
- "description": "Rate a reply in one of the caller's chats.\n\nSaves the rating, one per reply and score name so rating again\noverwrites it, and scores it in Langfuse on the trace of the turn that\nwrote the reply, or on the chat's Langfuse session when that trace is\nnot known. Langfuse being down does not fail the request; the response\nsays where the score went.",
+ "description": "Rate a reply in one of the caller's chats.\n\nSaves the rating, one per reply and score name so rating again\noverwrites it, and scores it in Langfuse on the trace of the turn that\nwrote the reply, or on the chat's Langfuse session when that trace is\nnot known. It also goes to PostHog LLM analytics as an ``$ai_metric``\nevent. Langfuse or PostHog being down does not fail the request; the\nresponse says where the Langfuse score went.",
"operationId": "postV2Submit message feedback",
"security": [{ "HTTPBearerJWT": [] }],
"parameters": [
diff --git a/docs/engineering/tracking-plan.md b/docs/engineering/tracking-plan.md
index 50ded17b63..f8c04cdc8b 100644
--- a/docs/engineering/tracking-plan.md
+++ b/docs/engineering/tracking-plan.md
@@ -298,6 +298,45 @@ sent or dropped this way, not while it is held.
Return visits are `$pageview`; account-level retention is computed in
`retention_task_weekly` and `user_lifecycle`, not sent as events.
+## LLM analytics
+
+AutoPilot's LLM calls and the ratings of its replies, in PostHog's LLM
+analytics (SECRT-2730). Both are sent server-side for every user, whatever
+their cookie consent, like every backend event. The event names are
+PostHog's own, not the analytics plan's.
+
+| Event | Sender | Status | Required properties | Fires when |
+| --- | --- | --- | --- | --- |
+| `$ai_generation` | OpenRouter Broadcast | live | set by OpenRouter from the request (`backend/copilot/llm_attribution.py`): `distinct_id` = the request's `user` (the platform user id), `$ai_session_id` = its `session_id` (the chat session id), `$ai_span_name` = `trace.generation_name` (`copilot-title`, `copilot-sdk`, `copilot-baseline`), `metadata_environment` (`local`, `dev`, `prod`), `metadata_impersonated`; model, tokens, cost and latency from OpenRouter | An AutoPilot LLM call goes through OpenRouter: a session title, an SDK turn (main agent, sub-agents and the CLI's compaction, via `CLAUDE_CODE_EXTRA_BODY`) or a baseline turn. |
+| `$ai_metric` | backend | live | `$ai_metric_name` (`user-feedback`, `copy`), `$ai_metric_value` (`1` thumbs up or copy, `0` thumbs down), `$ai_session_id` and `chat_session_id` (the chat session id), `message_id` (the reply's row id), `message_sequence`, `feedback_id` (the stored rating, shared with its Langfuse score), `impersonated`; `$ai_feedback_text` when a thumbs-down has a comment | A reply is rated (`POST /api/chat/sessions/{id}/feedback`, `copilot/feedback.py`). Rating a reply again sends another event; the latest per `feedback_id` stands. |
+
+What joins them:
+
+- **Person.** A generation's `distinct_id` and a rating's are both the
+ platform user id, the id every other event and `posthog.identify` use, so
+ both land on the person with their product events and replays.
+- **Chat.** `$ai_session_id` is the chat session id on both, so a chat's
+ generations and its ratings group in the Sessions tab and join on it.
+ `message_id` says which reply was rated; a rating carries no
+ `$ai_trace_id`, since Broadcast assigns the generations' trace ids.
+- **Environment.** OpenRouter forwards our `trace` keys as `metadata_<key>`,
+ so a generation's environment is `metadata_environment`, not the
+ `environment` base property; `$ai_metric` has both base properties as usual.
+- **Impersonation.** A turn or rating an admin made while impersonating the
+ user (`X-Act-As-User-Id`) is still sent, as the user, with
+ `metadata_impersonated` / `impersonated` true, so it can be filtered out.
+ The mark is kept for a turn queued behind the running-turn cap and for
+ sub-sessions such a turn starts. A follow-up sent while a turn runs takes
+ that turn's mark, and an engine-switch continuation or an approval wake
+ starts unmarked.
+
+Calls that skip OpenRouter never reach Broadcast and have no
+`$ai_generation`: direct Anthropic and Claude Code subscription deployments,
+a user's connected Codex account and the local transport. Auxiliary calls
+that are not attributed yet (the baseline path's context compaction, the
+action gate's classifiers, web search, Graphiti memory) arrive without the
+user or chat when their client goes through OpenRouter.
+
## Experiments
| Event | Sender | Status | Required properties | Fires when |
@@ -483,5 +522,5 @@ so nobody adds a duplicate:
- `$pageleave`, `$autocapture`, `$identify` and `$feature_flag_called`:
posthog-js (`capture_pageleave` and `autocapture` are on).
-- `$ai_generation`: OpenRouter's PostHog integration, for copilot title
- generation (`posthogDistinctId` in `copilot/service.py`).
+- `$ai_generation`: OpenRouter Broadcast, for every AutoPilot LLM call that
+ goes through OpenRouter (see [LLM analytics](#llm-analytics)).