Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

📊 feat: Surface Message Feedback as Langfuse Scores (#13544)

Gil Raphaelli90ebecb25489
diff --git a/api/server/routes/messages.js b/api/server/routes/messages.js
index a07293c0e2..0133fe8541 100644
--- a/api/server/routes/messages.js
+++ b/api/server/routes/messages.js
@@ -1,8 +1,13 @@
const express = require('express');
const { v4: uuidv4 } = require('uuid');
const { logger } = require('@librechat/data-schemas');
-const { ContentTypes } = require('librechat-data-provider');
-const { unescapeLaTeX, countTokens } = require('@librechat/api');
+const { ContentTypes, isAssistantsEndpoint } = require('librechat-data-provider');
+const {
+ unescapeLaTeX,
+ countTokens,
+ sendFeedbackScore,
+ traceIdForMessage,
+} = require('@librechat/api');
const { findAllArtifacts, replaceArtifactContent } = require('~/server/services/Artifacts/update');
const { requireJwtAuth, validateMessageReq } = require('~/server/middleware');
const db = require('~/models');
@@ -391,6 +396,14 @@ router.put('/:conversationId/:messageId/feedback', validateMessageReq, async (re
{ context: 'updateFeedback' },
);
+ // Best-effort: Assistants messages do not have deterministic AgentRun traces.
+ if (!isAssistantsEndpoint(updatedMessage.endpoint)) {
+ sendFeedbackScore({
+ traceId: traceIdForMessage(messageId),
+ feedback: updatedMessage.feedback,
+ }).catch((err) => logger.error('[langfuse] feedback score failed:', err));
+ }
+
res.json({
messageId,
conversationId,
diff --git a/packages/api/src/agents/run.ts b/packages/api/src/agents/run.ts
index 2549ec7984..07aa726c01 100644
--- a/packages/api/src/agents/run.ts
+++ b/packages/api/src/agents/run.ts
@@ -1040,6 +1040,11 @@ export async function createRun({
calibrationRatio,
indexTokenCountMap,
eagerEventToolExecution: { enabled: true },
+ // Derive the Langfuse trace id deterministically from runId so message
+ // feedback can be scored against the trace without a lookup (see the
+ // feedback route in api/server/routes/messages.js). No-op unless Langfuse
+ // tracing is enabled. Requires @librechat/agents >= 3.2.21.
+ langfuse: { deterministicTraceId: true },
...(enableToolOutputReferences && {
toolOutputReferences: { enabled: true },
}),
diff --git a/packages/api/src/index.ts b/packages/api/src/index.ts
index 49e4ff50d3..d923a4e75d 100644
--- a/packages/api/src/index.ts
+++ b/packages/api/src/index.ts
@@ -57,6 +57,8 @@ export * from './storage';
export * from './tools';
/* web search */
export * from './web';
+/* Langfuse */
+export * from './langfuse';
/* Cache */
export * from './cache';
/* Shared Links */
diff --git a/packages/api/src/langfuse/feedback.spec.ts b/packages/api/src/langfuse/feedback.spec.ts
new file mode 100644
index 0000000000..daf64cf09e
--- /dev/null
+++ b/packages/api/src/langfuse/feedback.spec.ts
@@ -0,0 +1,125 @@
+jest.mock(
+ '@librechat/data-schemas',
+ () => ({
+ logger: {
+ debug: jest.fn(),
+ },
+ }),
+ { virtual: true },
+);
+
+const langfuseEnvKeys = [
+ 'LANGFUSE_PUBLIC_KEY',
+ 'LANGFUSE_SECRET_KEY',
+ 'LANGFUSE_BASE_URL',
+ 'LANGFUSE_HOST',
+ 'LANGFUSE_BASEURL',
+ 'LANGFUSE_TRACING_ENABLED',
+ 'LANGFUSE_SAMPLE_RATE',
+ 'LANGFUSE_TRACING_ENVIRONMENT',
+];
+let fetchMock: jest.SpiedFunction<typeof fetch>;
+
+function clearLangfuseEnv() {
+ for (const key of langfuseEnvKeys) {
+ delete process.env[key];
+ }
+}
+
+function setLangfuseCredentials() {
+ process.env.LANGFUSE_PUBLIC_KEY = 'public-key';
+ process.env.LANGFUSE_SECRET_KEY = 'secret-key';
+}
+
+async function loadFeedback(): Promise<typeof import('./feedback')> {
+ jest.resetModules();
+ return import('./feedback');
+}
+
+function getFetchMock(): jest.SpiedFunction<typeof fetch> {
+ return fetchMock;
+}
+
+describe('Langfuse feedback scores', () => {
+ beforeEach(() => {
+ clearLangfuseEnv();
+ setLangfuseCredentials();
+ fetchMock = jest.spyOn(global, 'fetch').mockResolvedValue(new Response(null, { status: 200 }));
+ });
+
+ afterEach(() => {
+ clearLangfuseEnv();
+ fetchMock.mockRestore();
+ jest.clearAllMocks();
+ });
+
+ it('posts feedback scores when Langfuse tracing is enabled by default', async () => {
+ const { sendFeedbackScore } = await loadFeedback();
+
+ await sendFeedbackScore({
+ traceId: 'trace-id',
+ feedback: { rating: 'thumbsUp', tag: 'helpful', text: 'nice' },
+ });
+
+ expect(getFetchMock()).toHaveBeenCalledWith(
+ 'https://cloud.langfuse.com/api/public/scores',
+ expect.objectContaining({
+ method: 'POST',
+ headers: expect.objectContaining({
+ Authorization: `Basic ${Buffer.from('public-key:secret-key').toString('base64')}`,
+ 'Content-Type': 'application/json',
+ }),
+ body: expect.any(String),
+ }),
+ );
+ const [, init] = getFetchMock().mock.calls[0];
+ expect(JSON.parse(init?.body as string)).toMatchObject({
+ id: 'feedback-trace-id',
+ traceId: 'trace-id',
+ name: 'user-feedback',
+ value: 1,
+ dataType: 'BOOLEAN',
+ comment: 'helpful — nice',
+ metadata: { rating: 'thumbsUp', tag: 'helpful' },
+ });
+ });
+
+ it('posts feedback scores to the configured Langfuse host', async () => {
+ process.env.LANGFUSE_HOST = 'http://langfuse-server:3000';
+ const { sendFeedbackScore } = await loadFeedback();
+
+ await sendFeedbackScore({
+ traceId: 'trace-id',
+ feedback: { rating: 'thumbsUp' },
+ });
+
+ expect(getFetchMock()).toHaveBeenCalledWith(
+ 'http://langfuse-server:3000/api/public/scores',
+ expect.objectContaining({ method: 'POST' }),
+ );
+ });
+
+ it('skips scores when Langfuse tracing is disabled', async () => {
+ process.env.LANGFUSE_TRACING_ENABLED = 'false';
+ const { sendFeedbackScore } = await loadFeedback();
+
+ await sendFeedbackScore({
+ traceId: 'trace-id',
+ feedback: { rating: 'thumbsDown' },
+ });
+
+ expect(getFetchMock()).not.toHaveBeenCalled();
+ });
+
+ it('skips scores when Langfuse sampling is set to zero', async () => {
+ process.env.LANGFUSE_SAMPLE_RATE = '0';
+ const { sendFeedbackScore } = await loadFeedback();
+
+ await sendFeedbackScore({
+ traceId: 'trace-id',
+ feedback: { rating: 'thumbsUp' },
+ });
+
+ expect(getFetchMock()).not.toHaveBeenCalled();
+ });
+});
diff --git a/packages/api/src/langfuse/feedback.ts b/packages/api/src/langfuse/feedback.ts
new file mode 100644
index 0000000000..db6e2abdc0
--- /dev/null
+++ b/packages/api/src/langfuse/feedback.ts
@@ -0,0 +1,86 @@
+import { logger } from '@librechat/data-schemas';
+
+export type LangfuseFeedback = {
+ rating?: 'thumbsUp' | 'thumbsDown';
+ tag?: string;
+ text?: string;
+};
+
+export type SendFeedbackScoreParams = {
+ traceId: string;
+ feedback?: LangfuseFeedback | null;
+};
+
+const DEFAULT_BASE_URL = 'https://cloud.langfuse.com';
+const BASE =
+ process.env.LANGFUSE_BASE_URL ??
+ process.env.LANGFUSE_HOST ??
+ process.env.LANGFUSE_BASEURL ??
+ DEFAULT_BASE_URL;
+
+function isFalseEnv(value?: string): boolean {
+ return value != null && ['0', 'false', 'no', 'off'].includes(value.trim().toLowerCase());
+}
+
+function isSampleRateEnabled(value?: string): boolean {
+ if (value == null || value.trim() === '') {
+ return true;
+ }
+ const parsed = Number(value);
+ return !Number.isFinite(parsed) || parsed !== 0;
+}
+
+const ENABLED =
+ Boolean(process.env.LANGFUSE_PUBLIC_KEY && process.env.LANGFUSE_SECRET_KEY) &&
+ !isFalseEnv(process.env.LANGFUSE_TRACING_ENABLED) &&
+ isSampleRateEnabled(process.env.LANGFUSE_SAMPLE_RATE);
+const AUTHORIZATION = ENABLED
+ ? 'Basic ' +
+ Buffer.from(`${process.env.LANGFUSE_PUBLIC_KEY}:${process.env.LANGFUSE_SECRET_KEY}`).toString(
+ 'base64',
+ )
+ : undefined;
+const ENVIRONMENT = process.env.LANGFUSE_TRACING_ENVIRONMENT;
+
+export async function sendFeedbackScore({
+ traceId,
+ feedback,
+}: SendFeedbackScoreParams): Promise<void> {
+ if (!ENABLED || !AUTHORIZATION || !traceId) {
+ return;
+ }
+
+ const scoreId = `feedback-${traceId}`;
+
+ if (!feedback?.rating) {
+ const res = await fetch(`${BASE}/api/public/scores/${encodeURIComponent(scoreId)}`, {
+ method: 'DELETE',
+ headers: { Authorization: AUTHORIZATION },
+ });
+ if (!res.ok && res.status !== 404) {
+ throw new Error(`langfuse score delete ${res.status}: ${await res.text()}`);
+ }
+ return;
+ }
+
+ const body = {
+ id: scoreId,
+ traceId,
+ name: 'user-feedback',
+ value: feedback.rating === 'thumbsUp' ? 1 : 0,
+ dataType: 'BOOLEAN',
+ comment: [feedback.tag, feedback.text].filter(Boolean).join(' — ') || undefined,
+ metadata: { rating: feedback.rating, tag: feedback.tag },
+ ...(ENVIRONMENT ? { environment: ENVIRONMENT } : {}),
+ };
+
+ const res = await fetch(`${BASE}/api/public/scores`, {
+ method: 'POST',
+ headers: { Authorization: AUTHORIZATION, 'Content-Type': 'application/json' },
+ body: JSON.stringify(body),
+ });
+ if (!res.ok) {
+ throw new Error(`langfuse score create ${res.status}: ${await res.text()}`);
+ }
+ logger.debug(`[langfuse] feedback score sent for trace ${traceId} (${feedback.rating})`);
+}
diff --git a/packages/api/src/langfuse/index.ts b/packages/api/src/langfuse/index.ts
new file mode 100644
index 0000000000..b22dba4698
--- /dev/null
+++ b/packages/api/src/langfuse/index.ts
@@ -0,0 +1,2 @@
+export * from './feedback';
+export * from './trace';
diff --git a/packages/api/src/langfuse/trace.ts b/packages/api/src/langfuse/trace.ts
new file mode 100644
index 0000000000..0bc80633c8
--- /dev/null
+++ b/packages/api/src/langfuse/trace.ts
@@ -0,0 +1,5 @@
+import { createHash } from 'crypto';
+
+export function traceIdForMessage(messageId: string): string {
+ return createHash('sha256').update(messageId, 'utf8').digest('hex').slice(0, 32);
+}
diff --git a/packages/data-schemas/src/methods/message.ts b/packages/data-schemas/src/methods/message.ts
index dc3ab6d5b2..9ee15c7978 100644
--- a/packages/data-schemas/src/methods/message.ts
+++ b/packages/data-schemas/src/methods/message.ts
@@ -285,6 +285,7 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa
isCreatedByUser: updatedMessage.isCreatedByUser,
tokenCount: updatedMessage.tokenCount,
feedback: updatedMessage.feedback,
+ endpoint: updatedMessage.endpoint,
};
} catch (err) {
logger.error('Error updating message:', err);