Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(platform/copilot): save thumbs up/down feedback as Langfuse scores (#14899)

Nicholas Tindle13d7de7a226e
diff --git a/autogpt_platform/backend/backend/api/features/chat/feedback.py b/autogpt_platform/backend/backend/api/features/chat/feedback.py
new file mode 100644
index 0000000000..d04f06f8a2
--- /dev/null
+++ b/autogpt_platform/backend/backend/api/features/chat/feedback.py
@@ -0,0 +1,96 @@
+"""Rating an AutoPilot reply: the thumbs up/down and copy actions under it.
+
+Served under ``/api/chat`` next to the other session routes; the rating
+logic lives in ``backend.copilot.feedback``.
+"""
+
+from typing import Annotated
+
+from autogpt_libs import auth
+from fastapi import APIRouter, HTTPException, Security
+from pydantic import BaseModel, Field, field_validator, model_validator
+
+from backend.copilot.feedback import (
+ MAX_FEEDBACK_COMMENT_CHARS,
+ FeedbackScoreName,
+ FeedbackScoreValue,
+ MessageFeedbackResponse,
+ record_message_feedback,
+)
+
+router = APIRouter(tags=["chat"])
+
+
+class MessageFeedbackRequest(BaseModel):
+ """One rating of one reply."""
+
+ message_id: str = Field(
+ min_length=1,
+ max_length=256,
+ description=(
+ "The reply as the chat UI knows it: ``<session id>-seq-<N>`` for "
+ "a reply loaded from the chat history, or the message's own id."
+ ),
+ )
+ score_name: FeedbackScoreName = Field(
+ description='"user-feedback" for thumbs up/down, "copy" when the '
+ "user copied the reply.",
+ )
+ score_value: FeedbackScoreValue = Field(
+ description="1 for thumbs up or a copy, 0 for thumbs down.",
+ )
+ comment: str | None = Field(
+ default=None,
+ max_length=MAX_FEEDBACK_COMMENT_CHARS,
+ description="What the user said was wrong, from the thumbs-down form.",
+ )
+
+ @field_validator("comment")
+ @classmethod
+ def blank_comment_is_none(cls, comment: str | None) -> str | None:
+ if comment is None:
+ return None
+ return comment.strip() or None
+
+ @model_validator(mode="after")
+ def copy_scores_one(self) -> "MessageFeedbackRequest":
+ if self.score_name == "copy" and self.score_value != 1:
+ raise ValueError("A copy is always scored 1")
+ return self
+
+
+ "/sessions/{session_id}/feedback",
+ summary="Submit message feedback",
+ dependencies=[Security(auth.requires_user)],
+ responses={
+ 404: {"description": "Session or reply not found, or not the caller's"},
+ },
+)
+async def submit_message_feedback(
+ session_id: str,
+ request: MessageFeedbackRequest,
+ user_id: Annotated[str, Security(auth.get_user_id)],
+) -> MessageFeedbackResponse:
+ """Rate a reply in one of the caller's chats.
+
+ Saves the rating, one per reply and score name so rating again
+ overwrites it, and scores it in Langfuse on the trace of the turn that
+ wrote the reply, or on the chat's Langfuse session when that trace is
+ not known. Langfuse being down does not fail the request; the response
+ says where the score went.
+ """
+ feedback = await record_message_feedback(
+ user_id=user_id,
+ session_id=session_id,
+ message_id=request.message_id,
+ score_name=request.score_name,
+ score_value=request.score_value,
+ comment=request.comment,
+ )
+ if feedback is None:
+ raise HTTPException(
+ status_code=404,
+ detail=f"Message {request.message_id} not found or access denied",
+ )
+ return feedback
diff --git a/autogpt_platform/backend/backend/api/features/chat/feedback_test.py b/autogpt_platform/backend/backend/api/features/chat/feedback_test.py
new file mode 100644
index 0000000000..fc88c67cc8
--- /dev/null
+++ b/autogpt_platform/backend/backend/api/features/chat/feedback_test.py
@@ -0,0 +1,369 @@
+"""Route tests for POST /sessions/{session_id}/feedback (thumbs up/down, copy).
+
+The database layer and the Langfuse client are mocked at the boundary;
+``backend/copilot/feedback_db_test.py`` proves the owner scoping against a
+real Postgres.
+"""
+
+from datetime import UTC, datetime
+from unittest.mock import AsyncMock, MagicMock
+
+import fastapi
+import fastapi.testclient
+import pytest
+import pytest_mock
+from langfuse.api import ScoreDataType
+from prisma.models import ChatMessage as PrismaChatMessage
+
+from backend.api.features.chat import feedback as feedback_routes
+from backend.copilot import feedback as feedback_service
+from backend.copilot import feedback_db
+
+app = fastapi.FastAPI()
+app.include_router(feedback_routes.router)
+client = fastapi.testclient.TestClient(app)
+
+SESSION_ID = "8d2f5c1e-2b7a-4c55-9d0e-3f6a1b2c4d5e"
+MESSAGE_ROW_ID = "0f9e8d7c-6b5a-4938-8271-605f4e3d2c1b"
+TRACE_ID = "1edf31f11b1693cc6103f358c1481694"
+FEEDBACK_ID = "c3a1b2d4-e5f6-4789-8abc-def012345678"
+
+
[email protected](autouse=True)
+def setup_app_auth(mock_jwt_user):
+ from autogpt_libs.auth.jwt_utils import get_jwt_payload
+
+ app.dependency_overrides[get_jwt_payload] = mock_jwt_user["get_jwt_payload"]
+ yield
+ app.dependency_overrides.clear()
+
+
[email protected](autouse=True)
+def langfuse_settings(mocker: pytest_mock.MockerFixture):
+ secrets = feedback_service.settings.secrets
+ mocker.patch.object(secrets, "langfuse_public_key", "pk-lf-test")
+ mocker.patch.object(secrets, "langfuse_secret_key", "sk-lf-test")
+ mocker.patch.object(secrets, "langfuse_tracing_environment", "dev")
+
+
+def score_create(mocker: pytest_mock.MockerFixture) -> AsyncMock:
+ """The Langfuse score API call, with the client it hangs off mocked."""
+ langfuse = MagicMock()
+ langfuse.async_api.score.create = AsyncMock()
+ mocker.patch.object(feedback_service, "get_client", return_value=langfuse)
+ return langfuse.async_api.score.create
+
+
+def upsert(mocker: pytest_mock.MockerFixture) -> AsyncMock:
+ return mocker.patch.object(
+ feedback_db,
+ "upsert_message_feedback",
+ new=AsyncMock(return_value=FEEDBACK_ID),
+ )
+
+
+def _reply(*, trace_id: str | None, sequence: int = 7) -> PrismaChatMessage:
+ return PrismaChatMessage(
+ id=MESSAGE_ROW_ID,
+ createdAt=datetime.now(UTC),
+ sessionId=SESSION_ID,
+ role="assistant",
+ content="Here is the report.",
+ sequence=sequence,
+ langfuseTraceId=trace_id,
+ )
+
+
+def _mock_message(
+ mocker: pytest_mock.MockerFixture, message: PrismaChatMessage | None
+) -> AsyncMock:
+ return mocker.patch.object(
+ feedback_db, "get_rateable_message", new=AsyncMock(return_value=message)
+ )
+
+
+def _post(body: dict, session_id: str = SESSION_ID):
+ return client.post(f"/sessions/{session_id}/feedback", json=body)
+
+
+def _thumbs_down(comment: str | None = "It edited the wrong file") -> dict:
+ return {
+ "message_id": f"{SESSION_ID}-seq-7",
+ "score_name": "user-feedback",
+ "score_value": 0,
+ "comment": comment,
+ }
+
+
+# ─── Access ────────────────────────────────────────────────────────────
+
+
+def test_feedback_rejects_unauthenticated_request() -> None:
+ unauthenticated_app = fastapi.FastAPI()
+ unauthenticated_app.include_router(feedback_routes.router)
+ unauthenticated_client = fastapi.testclient.TestClient(unauthenticated_app)
+
+ response = unauthenticated_client.post(
+ f"/sessions/{SESSION_ID}/feedback", json=_thumbs_down()
+ )
+
+ assert response.status_code == 401
+
+
+def test_feedback_on_another_users_session_is_not_found(
+ mocker: pytest_mock.MockerFixture,
+ test_user_id: str,
+ upsert: AsyncMock,
+ score_create: AsyncMock,
+) -> None:
+ """The lookup is scoped to the caller, so a session owned by someone else
+ yields no message and the same 404 an unknown session would."""
+ owner_id = "someone-else"
+
+ async def owner_only(user_id: str, session_id: str, **_):
+ return _reply(trace_id=TRACE_ID) if user_id == owner_id else None
+
+ lookup = mocker.patch.object(
+ feedback_db, "get_rateable_message", new=AsyncMock(side_effect=owner_only)
+ )
+
+ response = _post(_thumbs_down())
+
+ assert response.status_code == 404
+ assert lookup.await_args is not None
+ assert lookup.await_args.args == (test_user_id, SESSION_ID)
+ upsert.assert_not_awaited()
+ score_create.assert_not_awaited()
+
+
+def test_feedback_on_unknown_message_is_not_found(
+ mocker: pytest_mock.MockerFixture,
+ upsert: AsyncMock,
+ score_create: AsyncMock,
+) -> None:
+ _mock_message(mocker, None)
+
+ response = _post(_thumbs_down())
+
+ assert response.status_code == 404
+ assert "not found" in response.json()["detail"]
+ upsert.assert_not_awaited()
+ score_create.assert_not_awaited()
+
+
+# ─── Langfuse score ────────────────────────────────────────────────────
+
+
+def test_thumbs_down_scores_the_turn_trace(
+ mocker: pytest_mock.MockerFixture,
+ test_user_id: str,
+ upsert: AsyncMock,
+ score_create: AsyncMock,
+) -> None:
+ lookup = _mock_message(mocker, _reply(trace_id=TRACE_ID))
+
+ response = _post(_thumbs_down())
+
+ assert response.status_code == 200
+ assert response.json() == {"id": FEEDBACK_ID, "langfuse_target": "trace"}
+ lookup.assert_awaited_once_with(test_user_id, SESSION_ID, sequence=7)
+ upsert.assert_awaited_once_with(
+ user_id=test_user_id,
+ session_id=SESSION_ID,
+ message_id=MESSAGE_ROW_ID,
+ score_name="user-feedback",
+ score_value=0,
+ comment="It edited the wrong file",
+ langfuse_trace_id=TRACE_ID,
+ )
+ score_create.assert_awaited_once()
+ request = score_create.await_args.kwargs["request"]
+ assert request.id == FEEDBACK_ID
+ assert request.trace_id == TRACE_ID
+ assert request.session_id is None
+ assert request.name == "user-feedback"
+ assert request.value == 0
+ assert request.data_type == ScoreDataType.NUMERIC
+ assert request.comment == "It edited the wrong file"
+ assert request.metadata == {
+ "message_id": MESSAGE_ROW_ID,
+ "message_sequence": 7,
+ "session_id": SESSION_ID,
+ }
+ assert request.environment == "dev"
+
+
+def test_reply_without_a_trace_falls_back_to_a_session_score(
+ mocker: pytest_mock.MockerFixture,
+ upsert: AsyncMock,
+ score_create: AsyncMock,
+) -> None:
+ """Baseline turns and replies from before the trace stamp have no trace:
+ the score goes on the chat's Langfuse session, naming the message."""
+ _mock_message(mocker, _reply(trace_id=None))
+
+ response = _post(
+ {
+ "message_id": f"{SESSION_ID}-seq-7",
+ "score_name": "user-feedback",
+ "score_value": 1,
+ }
+ )
+
+ assert response.status_code == 200
+ assert response.json() == {"id": FEEDBACK_ID, "langfuse_target": "session"}
+ assert upsert.await_args is not None
+ assert upsert.await_args.kwargs["langfuse_trace_id"] is None
+ request = score_create.await_args.kwargs["request"]
+ assert request.trace_id is None
+ assert request.session_id == SESSION_ID
+ assert request.value == 1
+ assert request.comment is None
+ assert request.metadata["message_id"] == MESSAGE_ROW_ID
+ assert request.metadata["session_id"] == SESSION_ID
+
+
+def test_copy_is_scored_under_its_own_name(
+ mocker: pytest_mock.MockerFixture,
+ upsert: AsyncMock,
+ score_create: AsyncMock,
+) -> None:
+ _mock_message(mocker, _reply(trace_id=TRACE_ID))
+
+ response = _post(
+ {
+ "message_id": f"{SESSION_ID}-seq-7",
+ "score_name": "copy",
+ "score_value": 1,
+ }
+ )
+
+ assert response.status_code == 200
+ request = score_create.await_args.kwargs["request"]
+ assert request.name == "copy"
+ assert request.value == 1
+ assert request.trace_id == TRACE_ID
+
+
+def test_raw_row_id_is_looked_up_by_id(
+ mocker: pytest_mock.MockerFixture,
+ test_user_id: str,
+ upsert: AsyncMock,
+ score_create: AsyncMock,
+) -> None:
+ lookup = _mock_message(mocker, _reply(trace_id=TRACE_ID))
+
+ response = _post({**_thumbs_down(), "message_id": MESSAGE_ROW_ID})
+
+ assert response.status_code == 200
+ lookup.assert_awaited_once_with(test_user_id, SESSION_ID, message_id=MESSAGE_ROW_ID)
+
+
+def test_langfuse_failure_keeps_the_rating(
+ mocker: pytest_mock.MockerFixture,
+ upsert: AsyncMock,
+ score_create: AsyncMock,
+) -> None:
+ """Postgres already holds the rating, so a Langfuse outage is logged and
+ reported but does not fail the click."""
+ _mock_message(mocker, _reply(trace_id=TRACE_ID))
+ score_create.side_effect = RuntimeError("langfuse is down")
+
+ response = _post(_thumbs_down())
+
+ assert response.status_code == 200
+ assert response.json() == {"id": FEEDBACK_ID, "langfuse_target": None}
+ upsert.assert_awaited_once()
+
+
+def test_unconfigured_langfuse_saves_the_rating_only(
+ mocker: pytest_mock.MockerFixture,
+ upsert: AsyncMock,
+) -> None:
+ """Self-hosted installs without Langfuse still record ratings."""
+ _mock_message(mocker, _reply(trace_id=TRACE_ID))
+ mocker.patch.object(feedback_service.settings.secrets, "langfuse_public_key", "")
+ get_client = mocker.patch.object(feedback_service, "get_client")
+
+ response = _post(_thumbs_down())
+
+ assert response.status_code == 200
+ assert response.json() == {"id": FEEDBACK_ID, "langfuse_target": None}
+ upsert.assert_awaited_once()
+ get_client.assert_not_called()
+
+
+def test_database_failure_is_an_error_the_ui_can_show(
+ mocker: pytest_mock.MockerFixture,
+ score_create: AsyncMock,
+) -> None:
+ """With the rating unsaved the request fails rather than reporting a
+ success the UI would thank the user for."""
+ _mock_message(mocker, _reply(trace_id=TRACE_ID))
+ mocker.patch.object(
+ feedback_db,
+ "upsert_message_feedback",
+ new=AsyncMock(side_effect=RuntimeError("db down")),
+ )
+ failing_client = fastapi.testclient.TestClient(app, raise_server_exceptions=False)
+
+ response = failing_client.post(
+ f"/sessions/{SESSION_ID}/feedback", json=_thumbs_down()
+ )
+
+ assert response.status_code == 500
+ score_create.assert_not_awaited()
+
+
+# ─── Validation ────────────────────────────────────────────────────────
+
+
+ "overrides",
+ [
+ {"score_name": "stars"},
+ {"score_value": 2},
+ {"score_value": -1},
+ {"score_value": 0.5},
+ {"score_name": "copy", "score_value": 0},
+ {"comment": "x" * 2001},
+ {"message_id": ""},
+ ],
+ ids=[
+ "unknown-score-name",
+ "value-above-one",
+ "negative-value",
+ "fractional-value",
+ "copy-scored-zero",
+ "comment-too-long",
+ "empty-message-id",
+ ],
+)
+def test_invalid_ratings_are_rejected(
+ mocker: pytest_mock.MockerFixture,
+ overrides: dict,
+ upsert: AsyncMock,
+) -> None:
+ lookup = _mock_message(mocker, _reply(trace_id=TRACE_ID))
+
+ response = _post({**_thumbs_down(), **overrides})
+
+ assert response.status_code == 422
+ lookup.assert_not_awaited()
+ upsert.assert_not_awaited()
+
+
+def test_blank_comment_is_saved_as_none(
+ mocker: pytest_mock.MockerFixture,
+ upsert: AsyncMock,
+ score_create: AsyncMock,
+) -> None:
+ _mock_message(mocker, _reply(trace_id=TRACE_ID))
+
+ response = _post(_thumbs_down(comment=" "))
+
+ assert response.status_code == 200
+ assert upsert.await_args is not None
+ assert upsert.await_args.kwargs["comment"] is None
diff --git a/autogpt_platform/backend/backend/api/rest_api.py b/autogpt_platform/backend/backend/api/rest_api.py
index ff8db10e24..78ece733f1 100644
--- a/autogpt_platform/backend/backend/api/rest_api.py
+++ b/autogpt_platform/backend/backend/api/rest_api.py
@@ -36,6 +36,7 @@ import backend.api.features.blocks.routes as blocks_routes
import backend.api.features.briefings.routes
import backend.api.features.builder
import backend.api.features.builder.routes
+import backend.api.features.chat.feedback as chat_feedback
import backend.api.features.chat.routes as chat_routes
import backend.api.features.chat.share as chat_share
import backend.api.features.chat.speech as chat_speech
@@ -572,6 +573,11 @@ app.include_router(
tags=["v2", "chat"],
prefix="/api/chat",
)
+app.include_router(
+ chat_feedback.router,
+ tags=["v2", "chat"],
+ prefix="/api/chat",
+)
app.include_router(
chat_speech.router,
tags=["chat"],
diff --git a/autogpt_platform/backend/backend/copilot/db.py b/autogpt_platform/backend/backend/copilot/db.py
index 996cabeb86..4ed9fc64bb 100644
--- a/autogpt_platform/backend/backend/copilot/db.py
+++ b/autogpt_platform/backend/backend/copilot/db.py
@@ -724,6 +724,8 @@ async def add_chat_messages_batch(
data["llmAuthProvider"] = msg["llm_auth_provider"]
if msg.get("llm_credential_id") is not None:
data["llmCredentialId"] = msg["llm_credential_id"]
+ if msg.get("langfuse_trace_id") is not None:
+ data["langfuseTraceId"] = msg["langfuse_trace_id"]
# Per-row bag. The single-message path already persisted
# this; the batch path silently dropped it, so anything
@@ -1261,6 +1263,7 @@ async def update_chat_message_stamps(
routing_source: str | None,
llm_auth_provider: str | None = None,
llm_credential_id: str | None = None,
+ langfuse_trace_id: str | None = None,
) -> bool:
"""Back-fill the execution stamps on an already-persisted message row.
@@ -1269,9 +1272,10 @@ async def update_chat_message_stamps(
analytics columns survive in the DB. Same mechanism and authorization
reasoning as ``update_chat_message_tool_calls``.
- The route is written only when known. Passing None for it would blank a
- row that a previous stamp already got right, which is precisely the
- rewriting of history per-turn segments exist to prevent.
+ The route and the trace are written only when known. Passing None for
+ the route would blank a row that a previous stamp already got right,
+ which is precisely the rewriting of history per-turn segments exist to
+ prevent.
"""
data: ChatMessageUpdateInput = {
"model": model,
@@ -1281,6 +1285,8 @@ async def update_chat_message_stamps(
data["llmAuthProvider"] = llm_auth_provider
if llm_credential_id is not None:
data["llmCredentialId"] = llm_credential_id
+ if langfuse_trace_id is not None:
+ data["langfuseTraceId"] = langfuse_trace_id
result = await PrismaChatMessage.prisma().update(
where={"sessionId_sequence": {"sessionId": session_id, "sequence": sequence}},
data=data,
diff --git a/autogpt_platform/backend/backend/copilot/db_test.py b/autogpt_platform/backend/backend/copilot/db_test.py
index 35b562a596..18b413c265 100644
--- a/autogpt_platform/backend/backend/copilot/db_test.py
+++ b/autogpt_platform/backend/backend/copilot/db_test.py
@@ -1105,6 +1105,7 @@ async def test_batch_persist_maps_stamp_columns_to_prisma_names():
"content": "hi",
"model": "claude-sonnet-4-6",
"routing_source": "catalog",
+ "langfuse_trace_id": "1edf31f11b1693cc6103f358c1481694",
}
],
)
@@ -1113,6 +1114,8 @@ async def test_batch_persist_maps_stamp_columns_to_prisma_names():
assert row["model"] == "claude-sonnet-4-6"
assert row["routingSource"] == "catalog"
assert "routing_source" not in row
+ assert row["langfuseTraceId"] == "1edf31f11b1693cc6103f358c1481694"
+ assert "langfuse_trace_id" not in row
def test_from_db_restores_stamp_columns():
@@ -1127,11 +1130,15 @@ def test_from_db_restores_stamp_columns():
content="hi",
model="claude-sonnet-4-6",
routingSource="catalog",
+ langfuseTraceId="1edf31f11b1693cc6103f358c1481694",
createdAt=datetime.now(UTC),
)
restored = ChatMessage.from_db(prisma_msg)
assert restored.model == "claude-sonnet-4-6"
assert restored.routing_source == "catalog"
+ assert restored.langfuse_trace_id == "1edf31f11b1693cc6103f358c1481694"
+ # Internal: the trace id never reaches the session payload clients get.
+ assert "langfuse_trace_id" not in restored.model_dump()
def test_from_db_null_stamps_stay_null():
@@ -1178,6 +1185,36 @@ async def test_update_chat_message_stamps_maps_prisma_columns():
assert kwargs["where"]["sessionId_sequence"]["sequence"] == 7
+async def test_update_chat_message_stamps_writes_the_trace_only_when_known():
+ """The turn's trace back-fills with the other stamps; a row flagged for
+ another stamp with no trace known never has its trace blanked."""
+ from backend.copilot.db import update_chat_message_stamps
+
+ with patch.object(PrismaChatMessage, "prisma") as mock_msg:
+ update = AsyncMock(return_value=object())
+ mock_msg.return_value.update = update
+
+ await update_chat_message_stamps(
+ session_id=SESSION_ID,
+ sequence=7,
+ model="claude-sonnet-4-6",
+ routing_source="env",
+ langfuse_trace_id="1edf31f11b1693cc6103f358c1481694",
+ )
+ with_trace = update.call_args.kwargs["data"]
+ await update_chat_message_stamps(
+ session_id=SESSION_ID,
+ sequence=7,
+ model="claude-sonnet-4-6",
+ routing_source="env",
+ )
+ without_trace = update.call_args.kwargs["data"]
+
+ assert with_trace["langfuseTraceId"] == "1edf31f11b1693cc6103f358c1481694"
+ assert "langfuseTraceId" not in without_trace
+
+
@pytest.mark.asyncio
async def test_update_chat_message_stamps_not_found_returns_false():
"""No matching row → False (the back-fill loop logs and keeps going)."""
diff --git a/autogpt_platform/backend/backend/copilot/feedback.py b/autogpt_platform/backend/backend/copilot/feedback.py
new file mode 100644
index 0000000000..de565a89e3
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/feedback.py
@@ -0,0 +1,176 @@
+"""Thumbs up/down and copy ratings on AutoPilot replies.
+
+Postgres (``ChatMessageFeedback``) keeps every rating. Each one is also sent
+to Langfuse as a score named after the rating ("user-feedback": 1 = up,
+0 = down; "copy": 1), in the environment the copilot traces use:
+
+- on the trace of the SDK turn that wrote the reply, which the turn stamps
+ on its assistant rows (``ChatMessage.langfuseTraceId``);
+- otherwise on the chat's Langfuse session, with the message in the score's
+ metadata: baseline turns have no turn-level trace, and replies written
+ before the stamp existed have none recorded.
+
+The score reuses the rating's id, so rating a reply again updates its score
+instead of adding one. A Langfuse failure is logged and shows in the
+response; the rating itself is already saved.
+"""
+
+import logging
+from typing import Literal
+
+from langfuse import get_client
+from langfuse.api import CreateScoreRequest, ScoreDataType
+from prisma.models import ChatMessage as PrismaChatMessage
+from pydantic import BaseModel, Field
+
+from backend.copilot import feedback_db
+from backend.util.settings import Settings
+
+logger = logging.getLogger(__name__)
+
+settings = Settings()
+
+FeedbackScoreName = Literal["user-feedback", "copy"]
+FeedbackScoreValue = Literal[0, 1]
+LangfuseScoreTarget = Literal["trace", "session"]
+
+MAX_FEEDBACK_COMMENT_CHARS = 2000
+
+# The rating waits on Langfuse, so an unresponsive Langfuse must not hold the
+# click for long; the rating is saved regardless.
+_LANGFUSE_TIMEOUT_SECONDS = 5
+
+# Sequences are Postgres ``int4``: more digits than this could overflow the
+# lookup, and no chat is that long.
+_MAX_SEQUENCE_DIGITS = 9
+
+
+class MessageFeedbackResponse(BaseModel):
+ """A saved rating of an AutoPilot reply."""
+
+ id: str = Field(
+ description="The rating's id, which its Langfuse score shares.",
+ )
+ langfuse_target: LangfuseScoreTarget | None = Field(
+ default=None,
+ description=(
+ "Where the Langfuse score went: the trace of the turn that wrote "
+ "the reply, or the chat's Langfuse session when that trace is not "
+ "known. Null when Langfuse is not configured or did not take the "
+ "score; the rating is saved either way."
+ ),
+ )
+
+
+async def record_message_feedback(
+ *,
+ user_id: str,
+ session_id: str,
+ message_id: str,
+ score_name: FeedbackScoreName,
+ score_value: FeedbackScoreValue,
+ comment: str | None,
+) -> MessageFeedbackResponse | None:
+ """Save a rating of one reply in *user_id*'s chat and score it in Langfuse.
+
+ *message_id* is the id the chat UI knows the reply by: ``<session>-seq-<N>``
+ for a reply loaded from the chat history, or the message row's own id.
+
+ Returns ``None`` when the session is not the caller's or *message_id* is
+ not a reply in it.
+ """
+ message = await _find_rated_message(user_id, session_id, message_id)
+ if message is None:
+ return None
+ feedback_id = await feedback_db.upsert_message_feedback(
+ user_id=user_id,
+ session_id=session_id,
+ message_id=message.id,
+ score_name=score_name,
+ score_value=score_value,
+ comment=comment,
+ langfuse_trace_id=message.langfuseTraceId,
+ )
+ target = await _score_in_langfuse(
+ feedback_id=feedback_id,
+ session_id=session_id,
+ message=message,
+ score_name=score_name,
+ score_value=score_value,
+ comment=comment,
+ )
+ return MessageFeedbackResponse(id=feedback_id, langfuse_target=target)
+
+
+def sequence_from_ui_message_id(session_id: str, message_id: str) -> int | None:
+ """The row sequence a chat-UI message id names, if it names one.
+
+ The copilot UI keys a reply loaded from history ``<session>-seq-<N>``,
+ N being the sequence of the last row it folded into the bubble (see
+ ``convertChatSessionToUiMessages`` in the frontend).
+ """
+ digits = message_id.removeprefix(f"{session_id}-seq-")
+ if digits == message_id or not (digits.isascii() and digits.isdigit()):
+ return None
+ if len(digits) > _MAX_SEQUENCE_DIGITS:
+ return None
+ return int(digits)
+
+
+async def _find_rated_message(
+ user_id: str, session_id: str, message_id: str
+) -> PrismaChatMessage | None:
+ sequence = sequence_from_ui_message_id(session_id, message_id)
+ if sequence is not None:
+ return await feedback_db.get_rateable_message(
+ user_id, session_id, sequence=sequence
+ )
+ return await feedback_db.get_rateable_message(
+ user_id, session_id, message_id=message_id
+ )
+
+
+async def _score_in_langfuse(
+ *,
+ feedback_id: str,
+ session_id: str,
+ message: PrismaChatMessage,
+ score_name: FeedbackScoreName,
+ score_value: FeedbackScoreValue,
+ comment: str | None,
+) -> LangfuseScoreTarget | None:
+ """Send the rating to Langfuse and say where it landed, or None."""
+ if not (
+ settings.secrets.langfuse_public_key and settings.secrets.langfuse_secret_key
+ ):
+ return None
+ trace_id = message.langfuseTraceId
+ request = CreateScoreRequest(
+ id=feedback_id,
+ # A score belongs to a trace or to a session, not both.
+ traceId=trace_id,
+ sessionId=None if trace_id else session_id,
+ name=score_name,
+ value=score_value,
+ dataType=ScoreDataType.NUMERIC,
+ comment=comment,
+ metadata={
+ "message_id": message.id,
+ "message_sequence": message.sequence,
+ "session_id": session_id,
+ },
+ environment=settings.secrets.langfuse_tracing_environment,
+ )
+ try:
+ await get_client().async_api.score.create(
+ request=request,
+ request_options={"timeout_in_seconds": _LANGFUSE_TIMEOUT_SECONDS},
+ )
+ except Exception:
+ logger.warning(
+ f"Langfuse did not take {score_name} score {feedback_id} for "
+ f"session {session_id}; the rating is saved in Postgres only",
+ exc_info=True,
+ )
+ return None
+ return "trace" if trace_id else "session"
diff --git a/autogpt_platform/backend/backend/copilot/feedback_db.py b/autogpt_platform/backend/backend/copilot/feedback_db.py
new file mode 100644
index 0000000000..a3ca039657
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/feedback_db.py
@@ -0,0 +1,101 @@
+"""Database access for ratings on AutoPilot replies.
+
+Every query is scoped to the caller: a message is only found through a chat
+session ``user_id`` owns, and a rating row is keyed by ``user_id``, so a
+caller can neither read nor rate another user's replies even if a route
+forgot its own ownership check.
+"""
+
+from prisma.errors import UniqueViolationError
+from prisma.models import ChatMessage as PrismaChatMessage
+from prisma.models import ChatMessageFeedback
+from prisma.types import (
+ ChatMessageFeedbackCreateInput,
+ ChatMessageFeedbackUpdateInput,
+ ChatMessageFeedbackUpsertInput,
+ ChatMessageFeedbackWhereUniqueInput,
+ ChatMessageWhereInput,
+)
+
+from backend.util.json import sanitize_string
+
+# The rows a reply bubble in the chat UI can be keyed by: the UI folds a
+# turn's assistant and reasoning rows into one bubble and names it after the
+# last of them. User, tool and system rows are never rated.
+RATEABLE_ROLES = ("assistant", "reasoning")
+
+
+async def get_rateable_message(
+ user_id: str,
+ session_id: str,
+ *,
+ message_id: str | None = None,
+ sequence: int | None = None,
+) -> PrismaChatMessage | None:
+ """The reply row in *user_id*'s session, found by id or by sequence.
+
+ Returns ``None`` when the session is not the caller's, when no such row
+ exists in it, or when the row is not a reply.
+ """
+ if (message_id is None) == (sequence is None):
+ raise ValueError("Pass exactly one of message_id and sequence")
+ where: ChatMessageWhereInput = {
+ "sessionId": session_id,
+ "Session": {"is": {"userId": user_id}},
+ "role": {"in": list(RATEABLE_ROLES)},
+ }
+ if message_id is not None:
+ where["id"] = message_id
+ if sequence is not None:
+ where["sequence"] = sequence
+ return await PrismaChatMessage.prisma().find_first(where=where)
+
+
+async def upsert_message_feedback(
+ *,
+ user_id: str,
+ session_id: str,
+ message_id: str,
+ score_name: str,
+ score_value: int,
+ comment: str | None,
+ langfuse_trace_id: str | None,
+) -> str:
+ """Save *user_id*'s rating of a reply and return the rating's id.
+
+ One row per user, message and score name: rating again overwrites the
+ value and comment and keeps the id, which is also the Langfuse score id.
+ The caller must already have checked that the message is in a session
+ *user_id* owns (see ``get_rateable_message``).
+ """
+ comment = sanitize_string(comment) if comment is not None else None
+ where: ChatMessageFeedbackWhereUniqueInput = {
+ "messageId_userId_scoreName": {
+ "messageId": message_id,
+ "userId": user_id,
+ "scoreName": score_name,
+ }
+ }
+ create: ChatMessageFeedbackCreateInput = {
+ "userId": user_id,
+ "messageId": message_id,
+ "sessionId": session_id,
+ "scoreName": score_name,
+ "scoreValue": score_value,
+ "comment": comment,
+ "langfuseTraceId": langfuse_trace_id,
+ }
+ update: ChatMessageFeedbackUpdateInput = {
+ "scoreValue": score_value,
+ "comment": comment,
+ "langfuseTraceId": langfuse_trace_id,
+ }
+ data: ChatMessageFeedbackUpsertInput = {"create": create, "update": update}
+ try:
+ row = await ChatMessageFeedback.prisma().upsert(where=where, data=data)
+ except UniqueViolationError:
+ # Two clicks raced on the first rating of this reply and the other
+ # one inserted the row between this upsert's read and its insert.
+ # It exists now, so the retry updates it.
+ row = await ChatMessageFeedback.prisma().upsert(where=where, data=data)
+ return row.id
diff --git a/autogpt_platform/backend/backend/copilot/feedback_db_test.py b/autogpt_platform/backend/backend/copilot/feedback_db_test.py
new file mode 100644
index 0000000000..5a0ae65093
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/feedback_db_test.py
@@ -0,0 +1,140 @@
+"""Tests for ``copilot.feedback_db`` against the real Postgres.
+
+The owner predicate is the whole authorization story for a rating, so it is
+proven against the database rather than a mocked query.
+"""
+
+from uuid import uuid4
+
+import pytest
+from prisma.models import ChatMessageFeedback
+
+from backend.copilot.db import add_chat_message, create_chat_session
+from backend.copilot.feedback_db import get_rateable_message, upsert_message_feedback
+
+
+async def _chat_with_reply(user_id: str) -> tuple[str, str]:
+ """A session owned by *user_id* holding a prompt (seq 0) and a reply
+ (seq 1). Returns the session id and the reply's row id."""
+ session = await create_chat_session(str(uuid4()), user_id)
+ await add_chat_message(session.session_id, "user", 0, content="Summarise it")
+ reply = await add_chat_message(
+ session.session_id, "assistant", 1, content="Here is the summary."
+ )
+ assert reply.id is not None
+ return session.session_id, reply.id
+
+
[email protected](loop_scope="session")
+async def test_reply_is_found_only_through_its_owners_session(
+ setup_test_user, test_user_id, setup_admin_user, admin_user_id
+):
+ session_id, reply_id = await _chat_with_reply(test_user_id)
+
+ by_sequence = await get_rateable_message(test_user_id, session_id, sequence=1)
+ by_id = await get_rateable_message(test_user_id, session_id, message_id=reply_id)
+ assert by_sequence is not None and by_sequence.id == reply_id
+ assert by_id is not None and by_id.id == reply_id
+
+ other_user = admin_user_id
+ assert await get_rateable_message(other_user, session_id, sequence=1) is None
+ assert (
+ await get_rateable_message(other_user, session_id, message_id=reply_id) is None
+ )
+
+
[email protected](loop_scope="session")
+async def test_only_replies_in_the_named_session_are_rateable(
+ setup_test_user, test_user_id
+):
+ session_id, reply_id = await _chat_with_reply(test_user_id)
+ other_session_id, _ = await _chat_with_reply(test_user_id)
+
+ # The prompt is the user's own words, not a reply.
+ assert await get_rateable_message(test_user_id, session_id, sequence=0) is None
+ assert await get_rateable_message(test_user_id, session_id, sequence=99) is None
+ # The same user's reply, named against a different chat.
+ assert (
+ await get_rateable_message(test_user_id, other_session_id, message_id=reply_id)
+ is None
+ )
+
+
[email protected](loop_scope="session")
+async def test_rating_again_updates_the_same_row(setup_test_user, test_user_id):
+ session_id, reply_id = await _chat_with_reply(test_user_id)
+
+ first = await upsert_message_feedback(
+ user_id=test_user_id,
+ session_id=session_id,
+ message_id=reply_id,
+ score_name="user-feedback",
+ score_value=0,
+ comment="Wrong numbers",
+ langfuse_trace_id=None,
+ )
+ second = await upsert_message_feedback(
+ user_id=test_user_id,
+ session_id=session_id,
+ message_id=reply_id,
+ score_name="user-feedback",
+ score_value=1,
+ comment=None,
+ langfuse_trace_id="1edf31f11b1693cc6103f358c1481694",
+ )
+
+ assert second == first
+ rows = await ChatMessageFeedback.prisma().find_many(where={"messageId": reply_id})
+ assert len(rows) == 1
+ assert rows[0].scoreValue == 1
+ assert rows[0].comment is None
+ assert rows[0].langfuseTraceId == "1edf31f11b1693cc6103f358c1481694"
+ assert rows[0].sessionId == session_id
+ assert rows[0].userId == test_user_id
+
+
[email protected](loop_scope="session")
+async def test_each_score_name_keeps_its_own_rating(setup_test_user, test_user_id):
+ session_id, reply_id = await _chat_with_reply(test_user_id)
+
+ vote = await upsert_message_feedback(
+ user_id=test_user_id,
+ session_id=session_id,
+ message_id=reply_id,
+ score_name="user-feedback",
+ score_value=1,
+ comment=None,
+ langfuse_trace_id=None,
+ )
+ copy = await upsert_message_feedback(
+ user_id=test_user_id,
+ session_id=session_id,
+ message_id=reply_id,
+ score_name="copy",
+ score_value=1,
+ comment=None,
+ langfuse_trace_id=None,
+ )
+
+ assert vote != copy
+
+
[email protected](loop_scope="session")
+async def test_comment_is_stripped_of_postgres_control_characters(
+ setup_test_user, test_user_id
+):
+ session_id, reply_id = await _chat_with_reply(test_user_id)
+
+ feedback_id = await upsert_message_feedback(
+ user_id=test_user_id,
+ session_id=session_id,
+ message_id=reply_id,
+ score_name="user-feedback",
+ score_value=0,
+ comment="bad\x00 answer",
+ langfuse_trace_id=None,
+ )
+
+ row = await ChatMessageFeedback.prisma().find_unique(where={"id": feedback_id})
+ assert row is not None
+ assert row.comment == "bad answer"
diff --git a/autogpt_platform/backend/backend/copilot/feedback_test.py b/autogpt_platform/backend/backend/copilot/feedback_test.py
new file mode 100644
index 0000000000..92ddc2d59e
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/feedback_test.py
@@ -0,0 +1,31 @@
+"""Tests for reading the chat UI's message ids in ``copilot.feedback``."""
+
+import pytest
+
+from backend.copilot.feedback import sequence_from_ui_message_id
+
+SESSION_ID = "8d2f5c1e-2b7a-4c55-9d0e-3f6a1b2c4d5e"
+
+
+ ("message_id", "expected"),
+ [
+ (f"{SESSION_ID}-seq-0", 0),
+ (f"{SESSION_ID}-seq-42", 42),
+ (f"{SESSION_ID}-seq-999999999", 999_999_999),
+ # A reply the UI has not re-keyed from the stream yet, or a row id.
+ ("5b0c2a8e-4f1d-4c3b-9a7e-6d5f4e3c2b1a", None),
+ # The idx fallback names a render position, not a row.
+ (f"{SESSION_ID}-idx-3", None),
+ # Another chat's reply never resolves against this one.
+ ("11111111-2222-4333-8444-555555555555-seq-3", None),
+ (f"{SESSION_ID}-seq-", None),
+ (f"{SESSION_ID}-seq-3a", None),
+ (f"{SESSION_ID}-seq--3", None),
+ (f"{SESSION_ID}-seq-²", None),
+ # Past int4: no chat is that long, and the lookup must not overflow.
+ (f"{SESSION_ID}-seq-2147483648", None),
+ ],
+)
+def test_sequence_from_ui_message_id(message_id: str, expected: int | None) -> None:
+ assert sequence_from_ui_message_id(SESSION_ID, message_id) == expected
diff --git a/autogpt_platform/backend/backend/copilot/model.py b/autogpt_platform/backend/backend/copilot/model.py
index 4255ebf05c..3a66d486ef 100644
--- a/autogpt_platform/backend/backend/copilot/model.py
+++ b/autogpt_platform/backend/backend/copilot/model.py
@@ -221,6 +221,11 @@ class ChatMessage(BaseModel):
llm_auth_provider: CopilotLlmAuthProvider | None = None
llm_credential_id: str | None = None
+ # Langfuse trace of the SDK turn that wrote this assistant or reasoning
+ # row; a rating of the reply is scored against it. Internal, so excluded
+ # from payloads like routing_source, and persisted the same way.
+ langfuse_trace_id: str | None = Field(default=None, exclude=True)
+
stamps_pending_save: bool = Field(default=False, exclude=True)
"""True when model/routing_source were stamped after this row was already
persisted (mid-turn flush assigned its sequence before end-of-turn
@@ -288,6 +293,7 @@ class ChatMessage(BaseModel):
"CopilotLlmAuthProvider | None", prisma_message.llmAuthProvider
),
llm_credential_id=prisma_message.llmCredentialId,
+ langfuse_trace_id=prisma_message.langfuseTraceId,
)
@@ -1161,6 +1167,7 @@ async def _save_session_to_db(
"routing_source": msg.routing_source,
"llm_auth_provider": msg.llm_auth_provider,
"llm_credential_id": msg.llm_credential_id,
+ "langfuse_trace_id": msg.langfuse_trace_id,
"metadata": msg.metadata,
}
)
@@ -1214,6 +1221,7 @@ async def _save_session_to_db(
routing_source=msg.routing_source,
llm_auth_provider=msg.llm_auth_provider,
llm_credential_id=msg.llm_credential_id,
+ langfuse_trace_id=msg.langfuse_trace_id,
)
except Exception as e:
logger.error(
diff --git a/autogpt_platform/backend/backend/copilot/model_test.py b/autogpt_platform/backend/backend/copilot/model_test.py
index ffefc98392..806b63561b 100644
--- a/autogpt_platform/backend/backend/copilot/model_test.py
+++ b/autogpt_platform/backend/backend/copilot/model_test.py
@@ -2082,10 +2082,53 @@ async def test_save_session_to_db_backfills_stamps_on_flushed_rows(
routing_source="env",
llm_auth_provider=None,
llm_credential_id=None,
+ langfuse_trace_id=None,
)
assert flushed.stamps_pending_save is False
[email protected](loop_scope="session")
+async def test_save_session_to_db_persists_the_turn_trace(
+ mocker: MockerFixture,
+) -> None:
+ """The Langfuse trace stamped at end of turn reaches the DB both on rows
+ flushed mid-turn (back-fill) and on rows inserted by this save."""
+ trace_id = "1edf31f11b1693cc6103f358c1481694"
+ flushed = ChatMessage(
+ role="assistant",
+ content="working on it",
+ sequence=7,
+ model="claude-sonnet-4-6",
+ routing_source="env",
+ langfuse_trace_id=trace_id,
+ stamps_pending_save=True,
+ )
+ unsaved = ChatMessage(
+ role="assistant",
+ content="done",
+ model="claude-sonnet-4-6",
+ routing_source="env",
+ langfuse_trace_id=trace_id,
+ )
+ session = _make_session_with_messages(flushed, unsaved)
+
+ mock_db = mocker.MagicMock()
+ mock_db.update_chat_session = mocker.AsyncMock()
+ mock_db.add_chat_messages_batch = mocker.AsyncMock(return_value=8)
+ mock_db.update_chat_message_stamps = mocker.AsyncMock(return_value=True)
+ mocker.patch("backend.copilot.model.chat_db", return_value=mock_db)
+
+ await _save_session_to_db(
+ session, existing_message_count=8, skip_existence_check=True
+ )
+
+ backfill = mock_db.update_chat_message_stamps.await_args
+ assert backfill.kwargs["sequence"] == 7
+ assert backfill.kwargs["langfuse_trace_id"] == trace_id
+ (inserted,) = mock_db.add_chat_messages_batch.await_args.kwargs["messages"]
+ assert inserted["langfuse_trace_id"] == trace_id
+
+
@pytest.mark.asyncio(loop_scope="session")
async def test_save_session_to_db_stamp_backfill_failure_keeps_flag(
mocker: MockerFixture,
diff --git a/autogpt_platform/backend/backend/copilot/sdk/service.py b/autogpt_platform/backend/backend/copilot/sdk/service.py
index 768d240828..d93aaface0 100644
--- a/autogpt_platform/backend/backend/copilot/sdk/service.py
+++ b/autogpt_platform/backend/backend/copilot/sdk/service.py
@@ -52,6 +52,7 @@ from backend.copilot.model_router import (
resolve_model_route,
)
from backend.copilot.budget_signal import build_turn_budget_block
+from backend.copilot.feedback_db import RATEABLE_ROLES
from backend.copilot.graphiti.context import fetch_warm_context
from backend.copilot.markers import append_error_marker
from backend.copilot.provider_failure import ProviderFailure
@@ -6267,6 +6268,11 @@ async def stream_chat_completion_sdk( # pyright: ignore[reportGeneralTypeIssues
actual_model=state.observed_model if state is not None else None,
routing_source=routing_source,
)
+ _stamp_turn_trace_id(
+ session.messages,
+ start_index=pre_turn_message_count,
+ trace_id=langfuse_trace_id,
+ )
# What this turn ran on, recorded on the turn rather than read
# back off the session later, so a route change cannot rewrite it.
stamp_segment(
@@ -6672,3 +6678,27 @@ def _stamp_turn_messages(
# save path back-fills the columns (insert only covers
# unsequenced rows).
msg.stamps_pending_save = True
+
+
+def _stamp_turn_trace_id(
+ messages: list[ChatMessage],
+ *,
+ start_index: int,
+ trace_id: str | None,
+) -> None:
+ """Record the turn's Langfuse trace on the reply rows it wrote.
+
+ A thumbs up/down on the reply is scored against this trace (see
+ ``backend.copilot.feedback``). Every rateable role is stamped: the UI
+ names a reply bubble after its last assistant *or* reasoning row, and a
+ rating of either must find the trace. Bounded to the turn and never
+ overwriting, exactly like ``_stamp_turn_messages``; rows flushed mid-turn
+ ride the same stamps back-fill.
+ """
+ if not trace_id:
+ return
+ for msg in messages[start_index:]:
+ if msg.role in RATEABLE_ROLES and msg.langfuse_trace_id is None:
+ msg.langfuse_trace_id = trace_id
+ if msg.sequence is not None:
+ msg.stamps_pending_save = True
diff --git a/autogpt_platform/backend/backend/copilot/sdk/service_helpers_test.py b/autogpt_platform/backend/backend/copilot/sdk/service_helpers_test.py
index 8dd7ca07cc..0d63e1b811 100644
--- a/autogpt_platform/backend/backend/copilot/sdk/service_helpers_test.py
+++ b/autogpt_platform/backend/backend/copilot/sdk/service_helpers_test.py
@@ -1690,6 +1690,84 @@ class TestStampTurnMessages:
), "pre_turn_message_count must be captured AFTER error-marker cleanup"
+class TestStampTurnTraceId:
+ """A thumbs up/down on a reply is scored against the trace recorded on
+ it, so the stamp must name this turn's trace on this turn's replies."""
+
+ TRACE_ID = "1edf31f11b1693cc6103f358c1481694"
+
+ def _messages(self):
+ from backend.copilot.model import ChatMessage
+
+ return [
+ ChatMessage(role="assistant", content="old", sequence=0),
+ ChatMessage(role="user", content="q", sequence=1),
+ ChatMessage(role="assistant", content="working", sequence=2),
+ ChatMessage(role="tool", content="{}", tool_call_id="t1"),
+ ChatMessage(role="assistant", content="done"),
+ # The UI can name the reply bubble after a trailing reasoning row.
+ ChatMessage(role="reasoning", content="checking the totals"),
+ ]
+
+ def test_stamps_this_turns_replies_only(self):
+ from backend.copilot.sdk.service import _stamp_turn_trace_id
+
+ msgs = self._messages()
+ _stamp_turn_trace_id(msgs, start_index=1, trace_id=self.TRACE_ID)
+
+ assert msgs[0].langfuse_trace_id is None # an earlier turn's reply
+ assert msgs[1].langfuse_trace_id is None # user row
+ assert msgs[2].langfuse_trace_id == self.TRACE_ID
+ assert msgs[3].langfuse_trace_id is None # tool row
+ assert msgs[4].langfuse_trace_id == self.TRACE_ID
+ assert msgs[5].langfuse_trace_id == self.TRACE_ID # reasoning row
+
+ def test_flushed_rows_flagged_for_backfill(self):
+ from backend.copilot.sdk.service import _stamp_turn_trace_id
+
+ msgs = self._messages()
+ _stamp_turn_trace_id(msgs, start_index=1, trace_id=self.TRACE_ID)
+
+ assert msgs[2].stamps_pending_save is True # flushed mid-turn
+ assert msgs[4].stamps_pending_save is False # inserted by this save
+
+ def test_never_overwrites_an_existing_trace(self):
+ from backend.copilot.sdk.service import _stamp_turn_trace_id
+
+ msgs = self._messages()
+ msgs[2].langfuse_trace_id = "a" * 32
+ _stamp_turn_trace_id(msgs, start_index=0, trace_id=self.TRACE_ID)
+
+ assert msgs[2].langfuse_trace_id == "a" * 32
+ assert msgs[2].stamps_pending_save is False
+
+ def test_no_trace_stamps_nothing(self):
+ """Langfuse off, or its span failed to open: nothing to record."""
+ from backend.copilot.sdk.service import _stamp_turn_trace_id
+
+ msgs = self._messages()
+ _stamp_turn_trace_id(msgs, start_index=0, trace_id=None)
+
+ assert all(m.langfuse_trace_id is None for m in msgs)
+ assert not any(m.stamps_pending_save for m in msgs)
+
+ def test_stamped_in_the_finally_after_the_trace_is_captured(self):
+ """The turn handler is too large to drive in a unit test, so pin at
+ the source that the stamp runs after the trace id is captured and
+ before the end-of-turn save that persists it."""
+ import inspect
+
+ from backend.copilot.sdk.service import stream_chat_completion_sdk
+
+ src = inspect.getsource(stream_chat_completion_sdk)
+ captured_at = src.index(
+ "langfuse_trace_id = get_client().get_current_trace_id()"
+ )
+ stamped_at = src.index("_stamp_turn_trace_id(")
+ saved_at = src.index("await asyncio.shield(upsert_chat_session(session))")
+ assert captured_at < stamped_at < saved_at
+
+
class TestSdkServingSegment:
def test_platform_turn_is_not_read_from_mutable_session_metadata(self):
from backend.copilot.sdk.service import _sdk_serving_segment
diff --git a/autogpt_platform/backend/migrations/20260925020000_add_chat_message_feedback/migration.sql b/autogpt_platform/backend/migrations/20260925020000_add_chat_message_feedback/migration.sql
new file mode 100644
index 0000000000..744afc921e
--- /dev/null
+++ b/autogpt_platform/backend/migrations/20260925020000_add_chat_message_feedback/migration.sql
@@ -0,0 +1,35 @@
+-- Thumbs up/down and copy ratings on AutoPilot replies, plus the Langfuse
+-- trace id of the SDK turn that wrote each assistant row so a rating can be
+-- scored against that trace.
+
+-- Nullable with no default: a catalog-only change, no table rewrite.
+-- AlterTable
+ALTER TABLE "ChatMessage" ADD COLUMN "langfuseTraceId" TEXT;
+
+-- CreateTable
+CREATE TABLE "ChatMessageFeedback" (
+ "id" TEXT NOT NULL,
+ "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ "userId" TEXT NOT NULL,
+ "messageId" TEXT NOT NULL,
+ "sessionId" TEXT NOT NULL,
+ "scoreName" TEXT NOT NULL,
+ "scoreValue" INTEGER NOT NULL,
+ "comment" TEXT,
+ "langfuseTraceId" TEXT,
+
+ CONSTRAINT "ChatMessageFeedback_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "ChatMessageFeedback_userId_idx" ON "ChatMessageFeedback"("userId");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "ChatMessageFeedback_messageId_userId_scoreName_key" ON "ChatMessageFeedback"("messageId", "userId", "scoreName");
+
+-- AddForeignKey
+ALTER TABLE "ChatMessageFeedback" ADD CONSTRAINT "ChatMessageFeedback_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "ChatMessageFeedback" ADD CONSTRAINT "ChatMessageFeedback_messageId_fkey" FOREIGN KEY ("messageId") REFERENCES "ChatMessage"("id") ON DELETE CASCADE ON UPDATE CASCADE;
diff --git a/autogpt_platform/backend/schema.prisma b/autogpt_platform/backend/schema.prisma
index 97c7376fd5..4f8358b357 100644
--- a/autogpt_platform/backend/schema.prisma
+++ b/autogpt_platform/backend/schema.prisma
@@ -149,6 +149,7 @@ model User {
CreditTransactions CreditTransaction[]
UserBalance UserBalance?
ChatSessions ChatSession[]
+ ChatMessageFeedback ChatMessageFeedback[]
UserBriefings UserBriefing[]
ActivityEvents ActivityEvent[]
AgentPresets AgentPreset[]
@@ -753,6 +754,12 @@ model ChatMessage {
llmAuthProvider String?
llmCredentialId String?
+ // Langfuse trace of the SDK turn that wrote this assistant or reasoning
+ // row, so a rating of the reply can be scored against that trace. Null on
+ // user/tool rows, on baseline turns (no turn-level trace) and on rows
+ // predating this column.
+ langfuseTraceId String?
+
// Per-row JSONB metadata bag. Currently holds the dispatcher's
// submit-time payload (file_ids, mode, model, permissions, context,
// request_arrival_at) on the user row that triggered a queued turn,
@@ -760,9 +767,41 @@ model ChatMessage {
// Generic so future per-row state can land here without a migration.
metadata Json?
+ Feedback ChatMessageFeedback[]
+
@@unique([sessionId, sequence])
}
+// A user's rating of an AutoPilot reply: thumbs up/down from the message
+// actions, or a copy of the reply. One row per user, message and score name,
+// so voting again updates the rating in place. The row id is also the id of
+// the matching Langfuse score, which makes re-sending a rating idempotent.
+model ChatMessageFeedback {
+ id String @id @default(uuid())
+ createdAt DateTime @default(now())
+ updatedAt DateTime @default(now()) @updatedAt
+
+ userId String
+ User User @relation(fields: [userId], references: [id], onDelete: Cascade)
+
+ messageId String
+ Message ChatMessage @relation(fields: [messageId], references: [id], onDelete: Cascade)
+
+ // Copied from the message so a chat's ratings read without a join.
+ sessionId String
+
+ scoreName String // "user-feedback" (1 = up, 0 = down) or "copy" (always 1)
+ scoreValue Int
+ comment String?
+
+ // The trace the Langfuse score was attached to. Null when the reply had
+ // none and the score went on the chat's Langfuse session instead.
+ langfuseTraceId String?
+
+ @@unique([messageId, userId, scoreName])
+ @@index([userId])
+}
+
// This model describes the Agent Graph/Flow (Multi Agent System).
model AgentGraph {
id String @default(uuid())
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/components/AssistantMessageActions.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/components/AssistantMessageActions.tsx
index 0262047e91..278697e4d2 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/components/AssistantMessageActions.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/components/AssistantMessageActions.tsx
@@ -39,13 +39,14 @@ export function AssistantMessageActions({
}: Props) {
const {
feedback,
+ canRate,
showFeedbackModal,
handleCopy,
handleUpvote,
handleDownvoteClick,
handleDownvoteSubmit,
handleDownvoteCancel,
- } = useMessageFeedback({ sessionID, messageID: message.id });
+ } = useMessageFeedback({ sessionID, message });
const text = extractTextFromParts(message.parts);
@@ -66,7 +67,7 @@ export function AssistantMessageActions({
onClick={handleUpvote}
variant="ghost"
size="icon-sm"
- disabled={feedback === "downvote"}
+ disabled={!canRate || feedback === "downvote"}
className={cn(
feedback === "upvote" && "text-green-300 hover:text-green-300",
feedback === "downvote" && "!opacity-20",
@@ -80,7 +81,7 @@ export function AssistantMessageActions({
onClick={handleDownvoteClick}
variant="ghost"
size="icon-sm"
- disabled={feedback === "upvote"}
+ disabled={!canRate || feedback === "upvote"}
className={cn(
feedback === "downvote" && "text-red-300 hover:text-red-300",
feedback === "upvote" && "!opacity-20",
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/components/__tests__/AssistantMessageActions.test.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/components/__tests__/AssistantMessageActions.test.tsx
new file mode 100644
index 0000000000..2a157b93e2
--- /dev/null
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/components/__tests__/AssistantMessageActions.test.tsx
@@ -0,0 +1,222 @@
+import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
+import { http, HttpResponse } from "msw";
+import type { UIDataTypes, UIMessage, UITools } from "ai";
+
+import { getPostV2SubmitMessageFeedbackMockHandler200 } from "@/app/api/__generated__/endpoints/chat/chat.msw";
+import type { MessageFeedbackRequest } from "@/app/api/__generated__/models/messageFeedbackRequest";
+import { server } from "@/mocks/mock-server";
+import {
+ cleanup,
+ fireEvent,
+ render,
+ screen,
+ waitFor,
+} from "@/tests/integrations/test-utils";
+import { AssistantMessageActions } from "../AssistantMessageActions";
+
+const { toastMock, captureExceptionMock } = vi.hoisted(() => ({
+ toastMock: vi.fn(),
+ captureExceptionMock: vi.fn(),
+}));
+
+vi.mock("@/components/molecules/Toast/use-toast", () => ({
+ toast: toastMock,
+ useToast: () => ({ toast: toastMock, dismiss: vi.fn(), toasts: [] }),
+}));
+
+vi.mock("@sentry/nextjs", async (importOriginal) => ({
+ ...(await importOriginal<typeof import("@sentry/nextjs")>()),
+ captureException: captureExceptionMock,
+ getTraceData: vi.fn(() => ({})),
+ withServerActionInstrumentation: vi.fn(
+ (_name: string, _options: unknown, callback: () => unknown) => callback(),
+ ),
+}));
+
+vi.mock("../TTSButton", () => ({
+ TTSButton: () => null,
+}));
+
+const SESSION_ID = "550e8400-e29b-41d4-a716-446655440000";
+const SAVED_MESSAGE_ID = `${SESSION_ID}-seq-7`;
+const FEEDBACK_URL = `/api/proxy/api/chat/sessions/${SESSION_ID}/feedback`;
+
+function reply(
+ id = SAVED_MESSAGE_ID,
+): UIMessage<unknown, UIDataTypes, UITools> {
+ return {
+ id,
+ role: "assistant",
+ parts: [{ type: "text", text: "Here is the summary.", state: "done" }],
+ };
+}
+
+function captureRequests() {
+ const bodies: MessageFeedbackRequest[] = [];
+ server.use(
+ getPostV2SubmitMessageFeedbackMockHandler200(async ({ request }) => {
+ bodies.push((await request.json()) as MessageFeedbackRequest);
+ return { id: "feedback-1", langfuse_target: "trace" };
+ }),
+ );
+ return bodies;
+}
+
+function failRequests() {
+ server.use(
+ http.post(FEEDBACK_URL, () =>
+ HttpResponse.json({ detail: "Internal Server Error" }, { status: 500 }),
+ ),
+ );
+}
+
+function toastTitles() {
+ return toastMock.mock.calls.map(
+ ([args]) => (args as { title: string }).title,
+ );
+}
+
+const upvoteButton = () =>
+ screen.getByRole("button", { name: "Good response" });
+const downvoteButton = () =>
+ screen.getByRole("button", { name: "Bad response" });
+
+beforeEach(() => {
+ Object.defineProperty(navigator, "clipboard", {
+ configurable: true,
+ value: { writeText: vi.fn(async (_text: string) => {}) },
+ });
+});
+
+afterEach(() => {
+ cleanup();
+ toastMock.mockReset();
+ captureExceptionMock.mockReset();
+});
+
+describe("AssistantMessageActions feedback", () => {
+ test("thanks the user for a thumbs up only once the rating is saved", async () => {
+ let releaseResponse: () => void = () => {};
+ const responseHeld = new Promise<void>((resolve) => {
+ releaseResponse = resolve;
+ });
+ const bodies: MessageFeedbackRequest[] = [];
+ server.use(
+ getPostV2SubmitMessageFeedbackMockHandler200(async ({ request }) => {
+ bodies.push((await request.json()) as MessageFeedbackRequest);
+ await responseHeld;
+ return { id: "feedback-1", langfuse_target: "trace" };
+ }),
+ );
+ render(
+ <AssistantMessageActions message={reply()} sessionID={SESSION_ID} />,
+ );
+
+ fireEvent.click(upvoteButton());
+
+ await waitFor(() => expect(bodies).toHaveLength(1));
+ expect(bodies[0]).toEqual({
+ message_id: SAVED_MESSAGE_ID,
+ score_name: "user-feedback",
+ score_value: 1,
+ });
+ expect(toastTitles()).not.toContain("Thank you for your feedback!");
+
+ releaseResponse();
+
+ await waitFor(() =>
+ expect(toastTitles()).toContain("Thank you for your feedback!"),
+ );
+ expect((downvoteButton() as HTMLButtonElement).disabled).toBe(true);
+ });
+
+ test("shows an error and resets the buttons when the rating is not saved", async () => {
+ failRequests();
+ render(
+ <AssistantMessageActions message={reply()} sessionID={SESSION_ID} />,
+ );
+
+ fireEvent.click(upvoteButton());
+
+ await waitFor(() =>
+ expect(toastMock).toHaveBeenCalledWith(
+ expect.objectContaining({
+ title: "Couldn't save your feedback",
+ variant: "destructive",
+ }),
+ ),
+ );
+ expect(toastTitles()).not.toContain("Thank you for your feedback!");
+ expect(captureExceptionMock).toHaveBeenCalled();
+ await waitFor(() =>
+ expect((downvoteButton() as HTMLButtonElement).disabled).toBe(false),
+ );
+
+ const bodies = captureRequests();
+ fireEvent.click(upvoteButton());
+ await waitFor(() => expect(bodies).toHaveLength(1));
+ });
+
+ test("sends the thumbs-down comment and thanks the user once saved", async () => {
+ const bodies = captureRequests();
+ render(
+ <AssistantMessageActions message={reply()} sessionID={SESSION_ID} />,
+ );
+
+ fireEvent.click(downvoteButton());
+ fireEvent.change(
+ await screen.findByPlaceholderText(
+ "Tell us what went wrong or could be improved...",
+ ),
+ { target: { value: "It edited the wrong file" } },
+ );
+ fireEvent.click(screen.getByRole("button", { name: "Submit feedback" }));
+
+ await waitFor(() =>
+ expect(toastTitles()).toContain("Thank you for your feedback!"),
+ );
+ expect(bodies).toEqual([
+ {
+ message_id: SAVED_MESSAGE_ID,
+ score_name: "user-feedback",
+ score_value: 0,
+ comment: "It edited the wrong file",
+ },
+ ]);
+ });
+
+ test("a reply still keyed by the stream's id cannot be rated yet", async () => {
+ const bodies = captureRequests();
+ render(
+ <AssistantMessageActions
+ message={reply("b7e3c1d2-streamed-reply")}
+ sessionID={SESSION_ID}
+ />,
+ );
+
+ expect((upvoteButton() as HTMLButtonElement).disabled).toBe(true);
+ expect((downvoteButton() as HTMLButtonElement).disabled).toBe(true);
+
+ fireEvent.click(screen.getByRole("button", { name: "Copy" }));
+
+ await waitFor(() => expect(toastTitles()).toContain("Copied!"));
+ expect(bodies).toEqual([]);
+ });
+
+ test("copying a saved reply records a copy without thanking the user", async () => {
+ const bodies = captureRequests();
+ render(
+ <AssistantMessageActions message={reply()} sessionID={SESSION_ID} />,
+ );
+
+ fireEvent.click(screen.getByRole("button", { name: "Copy" }));
+
+ await waitFor(() => expect(bodies).toHaveLength(1));
+ expect(bodies[0]).toEqual({
+ message_id: SAVED_MESSAGE_ID,
+ score_name: "copy",
+ score_value: 1,
+ });
+ expect(toastTitles()).toEqual(["Copied!"]);
+ });
+});
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/useMessageFeedback.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/useMessageFeedback.ts
index 718135a4dc..2a48581a82 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/useMessageFeedback.ts
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/useMessageFeedback.ts
@@ -1,47 +1,62 @@
+import { usePostV2SubmitMessageFeedback } from "@/app/api/__generated__/endpoints/chat/chat";
+import type { MessageFeedbackRequest } from "@/app/api/__generated__/models/messageFeedbackRequest";
import { toast } from "@/components/molecules/Toast/use-toast";
-import { environment } from "@/services/environment";
-import { getCopilotAuthHeaders } from "@/app/(platform)/copilot/helpers";
+import * as Sentry from "@sentry/nextjs";
+import type { UIDataTypes, UIMessage, UITools } from "ai";
import { useState } from "react";
+import { extractDbSequence } from "../../helpers/convertChatSessionToUiMessages";
+
+type Rating = "upvote" | "downvote";
interface Args {
sessionID: string | null;
- messageID: string;
+ message: UIMessage<unknown, UIDataTypes, UITools>;
}
-async function submitFeedbackToBackend(args: {
- sessionID: string;
- messageID: string;
- scoreName: string;
- scoreValue: number;
- comment?: string;
-}) {
- try {
- const authHeaders = await getCopilotAuthHeaders();
- await fetch(
- `${environment.getAGPTServerBaseUrl()}/api/chat/sessions/${args.sessionID}/feedback`,
- {
- method: "POST",
- headers: {
- "Content-Type": "application/json",
- ...authHeaders,
- },
- body: JSON.stringify({
- message_id: args.messageID,
- score_name: args.scoreName,
- score_value: args.scoreValue,
- comment: args.comment,
- }),
- },
- );
- } catch (err) {
- // Feedback submission is best-effort; silently ignore failures
- console.debug("[Copilot] Feedback submission failed:", err);
+export function useMessageFeedback({ sessionID, message }: Args) {
+ const [feedback, setFeedback] = useState<Rating | null>(null);
+ const [showFeedbackModal, setShowFeedbackModal] = useState(false);
+ const { mutateAsync: submitFeedback } = usePostV2SubmitMessageFeedback();
+
+ // A reply that just streamed still carries the stream's id until the chat
+ // reloads it with its saved one, and only the saved one can be rated.
+ const canRate = sessionID !== null && extractDbSequence(message) !== null;
+
+ async function sendScore(score: Omit<MessageFeedbackRequest, "message_id">) {
+ if (!sessionID) return false;
+ try {
+ await submitFeedback({
+ sessionId: sessionID,
+ data: { message_id: message.id, ...score },
+ });
+ return true;
+ } catch (error) {
+ Sentry.captureException(error);
+ return false;
+ }
}
-}
-export function useMessageFeedback({ sessionID, messageID }: Args) {
- const [feedback, setFeedback] = useState<"upvote" | "downvote" | null>(null);
- const [showFeedbackModal, setShowFeedbackModal] = useState(false);
+ async function saveRating(rating: Rating, comment?: string) {
+ const saved = await sendScore({
+ score_name: "user-feedback",
+ score_value: rating === "upvote" ? 1 : 0,
+ comment,
+ });
+ if (!saved) {
+ setFeedback(null);
+ toast({
+ title: "Couldn't save your feedback",
+ description: "Please try again.",
+ variant: "destructive",
+ });
+ return;
+ }
+ toast({
+ title: "Thank you for your feedback!",
+ variant: "success",
+ duration: 3000,
+ });
+ }
async function handleCopy(text: string) {
try {
@@ -55,51 +70,26 @@ export function useMessageFeedback({ sessionID, messageID }: Args) {
});
return;
}
- if (sessionID) {
- submitFeedbackToBackend({
- sessionID,
- messageID,
- scoreName: "copy",
- scoreValue: 1,
- });
- }
+ // A signal for us, not an action the user asked for: the copy itself
+ // succeeded, so a failure to record it is reported but not shown.
+ if (canRate) void sendScore({ score_name: "copy", score_value: 1 });
}
function handleUpvote() {
- if (feedback) return;
+ if (feedback || !canRate) return;
setFeedback("upvote");
- toast({
- title: "Thank you for your feedback!",
- variant: "success",
- duration: 3000,
- });
- if (sessionID) {
- submitFeedbackToBackend({
- sessionID,
- messageID,
- scoreName: "user-feedback",
- scoreValue: 1,
- });
- }
+ void saveRating("upvote");
}
function handleDownvoteClick() {
- if (feedback) return;
+ if (feedback || !canRate) return;
setFeedback("downvote");
setShowFeedbackModal(true);
}
function handleDownvoteSubmit(comment: string) {
setShowFeedbackModal(false);
- if (sessionID) {
- submitFeedbackToBackend({
- sessionID,
- messageID,
- scoreName: "user-feedback",
- scoreValue: 0,
- comment: comment || undefined,
- });
- }
+ void saveRating("downvote", comment || undefined);
}
function handleDownvoteCancel() {
@@ -109,6 +99,7 @@ export function useMessageFeedback({ sessionID, messageID }: Args) {
return {
feedback,
+ canRate,
showFeedbackModal,
handleCopy,
handleUpvote,
diff --git a/autogpt_platform/frontend/src/app/api/openapi.json b/autogpt_platform/frontend/src/app/api/openapi.json
index 37c368ea98..ab343c5493 100644
--- a/autogpt_platform/frontend/src/app/api/openapi.json
+++ b/autogpt_platform/frontend/src/app/api/openapi.json
@@ -4452,6 +4452,59 @@
}
}
},
+ "/api/chat/sessions/{session_id}/feedback": {
+ "post": {
+ "tags": ["v2", "chat", "chat"],
+ "summary": "Submit message feedback",
+ "description": "Rate a reply in one of the caller's chats.\n\nSaves the rating, one per reply and score name so rating again\noverwrites it, and scores it in Langfuse on the trace of the turn that\nwrote the reply, or on the chat's Langfuse session when that trace is\nnot known. Langfuse being down does not fail the request; the response\nsays where the score went.",
+ "operationId": "postV2Submit message feedback",
+ "security": [{ "HTTPBearerJWT": [] }],
+ "parameters": [
+ {
+ "name": "session_id",
+ "in": "path",
+ "required": true,
+ "schema": { "type": "string", "title": "Session Id" }
+ }
+ ],
+ "requestBody": {
+ "required": true,
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/MessageFeedbackRequest"
+ }
+ }
+ }
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/MessageFeedbackResponse"
+ }
+ }
+ }
+ },
+ "401": {
+ "$ref": "#/components/responses/HTTP401NotAuthenticatedError"
+ },
+ "404": {
+ "description": "Session or reply not found, or not the caller's"
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": { "$ref": "#/components/schemas/HTTPValidationError" }
+ }
+ }
+ }
+ }
+ }
+ },
"/api/chat/sessions/{session_id}/messages/pending": {
"get": {
"tags": ["v2", "chat", "chat"],
@@ -25797,6 +25850,62 @@
"required": ["query", "response"],
"title": "Message"
},
+ "MessageFeedbackRequest": {
+ "properties": {
+ "message_id": {
+ "type": "string",
+ "maxLength": 256,
+ "minLength": 1,
+ "title": "Message Id",
+ "description": "The reply as the chat UI knows it: ``<session id>-seq-<N>`` for a reply loaded from the chat history, or the message's own id."
+ },
+ "score_name": {
+ "type": "string",
+ "enum": ["user-feedback", "copy"],
+ "title": "Score Name",
+ "description": "\"user-feedback\" for thumbs up/down, \"copy\" when the user copied the reply."
+ },
+ "score_value": {
+ "type": "integer",
+ "enum": [0, 1],
+ "title": "Score Value",
+ "description": "1 for thumbs up or a copy, 0 for thumbs down."
+ },
+ "comment": {
+ "anyOf": [
+ { "type": "string", "maxLength": 2000 },
+ { "type": "null" }
+ ],
+ "title": "Comment",
+ "description": "What the user said was wrong, from the thumbs-down form."
+ }
+ },
+ "type": "object",
+ "required": ["message_id", "score_name", "score_value"],
+ "title": "MessageFeedbackRequest",
+ "description": "One rating of one reply."
+ },
+ "MessageFeedbackResponse": {
+ "properties": {
+ "id": {
+ "type": "string",
+ "title": "Id",
+ "description": "The rating's id, which its Langfuse score shares."
+ },
+ "langfuse_target": {
+ "anyOf": [
+ { "type": "string", "enum": ["trace", "session"] },
+ { "type": "null" }
+ ],
+ "title": "Langfuse Target",
+ "description": "Where the Langfuse score went: the trace of the turn that wrote the reply, or the chat's Langfuse session when that trace is not known. Null when Langfuse is not configured or did not take the score; the rating is saved either way."
+ }
+ },
+ "type": "object",
+ "required": ["id"],
+ "title": "MessageFeedbackResponse",
+ "description": "A saved rating of an AutoPilot reply."
+ },
"ModelInfo": {
"properties": {
"value": { "type": "string", "title": "Value" },