Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(scores): enhance POST /scores to support batch processing (#18408)

marliessophie570909dea0dc
diff --git a/fern/apis/client/definition/score.yml b/fern/apis/client/definition/score.yml
index 7f3ab20973..90d59114d9 100644
--- a/fern/apis/client/definition/score.yml
+++ b/fern/apis/client/definition/score.yml
@@ -5,12 +5,34 @@ service:
base-path: /api/public
endpoints:
create:
- docs: Add a score to the database, upserts on id
+ docs: >
+ Create scores asynchronously. Single score: 200 with ID. Batch: 202 if accepted,
+ or 207 with accepted/rejected counts and error messages. Do not automatically retry a 207 batch.
+ The generated reference shows only 200; batches return 202 or 207.
method: POST
path: /scores
- request: CreateScoreRequest
+ request: CreateScoresRequest
+ response: CreateScoresResponse
types:
+ CreateScoresRequest:
+ discriminated: false
+ union:
+ - CreateScoreRequest
+ - CreateScoreBatchRequest
+
+ CreateScoreBatchRequest:
+ type: list<CreateScoreRequest>
+ examples:
+ - value:
+ - name: "accuracy"
+ value: 0.9
+ traceId: "cdef-1234-5678-90ab"
+ - name: "helpfulness"
+ value: 1
+ dataType: "BOOLEAN"
+ traceId: "cdef-1234-5678-90ab"
+
CreateScoreRequest:
properties:
id: optional<string>
@@ -110,3 +132,32 @@ types:
- double
- string
docs: The value of the score. Must be passed as string for categorical scores, and numeric for boolean and numeric scores. For corrections pass value as string and set data type to CORRECTION
+
+ CreateScoresResponse:
+ discriminated: false
+ union:
+ - CreateScoreResponse
+ - CreateScoreBatchResponse
+ - CreateScoreBatchResults
+
+ CreateScoreBatchResponse:
+ docs: Batch accepted (HTTP 202).
+ properties:
+ message: literal<"Accepted">
+
+ CreateScoreResponse:
+ properties:
+ id:
+ type: string
+ docs: The id of the created object in Langfuse
+
+ CreateScoreBatchResults:
+ docs: Acceptance counts and errors (HTTP 207).
+ properties:
+ accepted: integer
+ rejected: integer
+ errors: list<CreateScoreBatchError>
+
+ CreateScoreBatchError:
+ properties:
+ message: string
diff --git a/fern/apis/server/definition/scores.yml b/fern/apis/server/definition/scores.yml
index f403b89817..847d0cc9f2 100644
--- a/fern/apis/server/definition/scores.yml
+++ b/fern/apis/server/definition/scores.yml
@@ -7,11 +7,14 @@ service:
base-path: /api/public
endpoints:
create:
- docs: Create a score (supports trace, observation, session, and dataset run scores)
+ docs: >
+ Create scores asynchronously. Single score: 200 with ID. Batch: 202 if accepted,
+ or 207 with accepted/rejected counts and error messages. Do not automatically retry a 207 batch.
+ The generated reference shows only 200; batches return 202 or 207.
method: POST
path: /scores
- request: CreateScoreRequest
- response: CreateScoreResponse
+ request: CreateScoresRequest
+ response: CreateScoresResponse
get-many:
availability:
status: deprecated
@@ -108,6 +111,24 @@ service:
response: commons.Score
types:
+ CreateScoresRequest:
+ discriminated: false
+ union:
+ - CreateScoreRequest
+ - CreateScoreBatchRequest
+
+ CreateScoreBatchRequest:
+ type: list<CreateScoreRequest>
+ examples:
+ - value:
+ - name: "accuracy"
+ value: 0.9
+ traceId: "cdef-1234-5678-90ab"
+ - name: "helpfulness"
+ value: 1
+ dataType: "BOOLEAN"
+ traceId: "cdef-1234-5678-90ab"
+
CreateScoreRequest:
properties:
id: optional<string>
@@ -196,6 +217,18 @@ types:
enum:
- API
- ANNOTATION
+ CreateScoresResponse:
+ discriminated: false
+ union:
+ - CreateScoreResponse
+ - CreateScoreBatchResponse
+ - CreateScoreBatchResults
+
+ CreateScoreBatchResponse:
+ docs: Batch accepted (HTTP 202).
+ properties:
+ message: literal<"Accepted">
+
CreateScoreResponse:
properties:
id:
@@ -256,3 +289,14 @@ types:
data: list<GetScoresResponseData>
meta: pagination.MetaResponse
_deprecation: optional<commons.Deprecation>
+
+ CreateScoreBatchResults:
+ docs: Acceptance counts and errors (HTTP 207).
+ properties:
+ accepted: integer
+ rejected: integer
+ errors: list<CreateScoreBatchError>
+
+ CreateScoreBatchError:
+ properties:
+ message: string
diff --git a/packages/shared/src/features/scores/interfaces/api/shared.ts b/packages/shared/src/features/scores/interfaces/api/shared.ts
index 4c9f3c00e9..752892af91 100644
--- a/packages/shared/src/features/scores/interfaces/api/shared.ts
+++ b/packages/shared/src/features/scores/interfaces/api/shared.ts
@@ -75,7 +75,7 @@ export const GetScoresQuery = z.object({
// POST /scores
// Roughly mirrors ScoreBody in `packages/shared/src/server/ingestion/types.ts`;
// keep them in sync for fields that cross both surfaces.
-export const PostScoresBody = applyScoreValidation(
+export const PostScoreBody = applyScoreValidation(
PostScoreBodyFoundationSchema.extend({
source: PublicApiCreateScoreSourceDomain.default(ScoreSourceEnum.API),
}).and(
@@ -120,7 +120,18 @@ export const PostScoresBody = applyScoreValidation(
path: ["configId"],
});
-export const PostScoresResponse = z.object({ id: z.string() });
+// Batch items are validated individually so invalid scores do not reject valid siblings.
+export const PostScoresBody = z.union([PostScoreBody, z.array(z.unknown())]);
+
+export const PostScoresResponse = z.union([
+ z.object({ id: z.string() }),
+ z.object({ message: z.string() }),
+ z.object({
+ accepted: z.number().int().nonnegative(),
+ rejected: z.number().int().nonnegative(),
+ errors: z.array(z.object({ message: z.string() })),
+ }),
+]);
// DELETE /scores/{scoreId}
// Please note that the DELETE /scores/{scoreId} endpoint supports all score types (trace, session, dataset run) across v1 and v2.
diff --git a/packages/shared/src/features/scores/interfaces/api/v1/endpoints.ts b/packages/shared/src/features/scores/interfaces/api/v1/endpoints.ts
index a1c4ccf6ee..901a7d266a 100644
--- a/packages/shared/src/features/scores/interfaces/api/v1/endpoints.ts
+++ b/packages/shared/src/features/scores/interfaces/api/v1/endpoints.ts
@@ -5,6 +5,7 @@ import {
DeleteScoreResponse,
GetScoreQuery,
GetScoresQuery,
+ PostScoreBody,
PostScoresBody,
PostScoresResponse,
} from "../shared";
@@ -39,6 +40,8 @@ export const GetScoresResponseV1 = z.object({
});
// POST /scores
+export const PostScoreBodyV1 = PostScoreBody;
+
export const PostScoresBodyV1 = PostScoresBody;
export const PostScoresResponseV1 = PostScoresResponse;
diff --git a/web/public/generated/api-client/openapi.yml b/web/public/generated/api-client/openapi.yml
index 1d5c7836d2..6fccf6c76a 100644
--- a/web/public/generated/api-client/openapi.yml
+++ b/web/public/generated/api-client/openapi.yml
@@ -5,14 +5,22 @@ info:
paths:
/api/public/scores:
post:
- description: Add a score to the database, upserts on id
+ description: >-
+ Create scores asynchronously. Single score: 200 with ID. Batch: 202 if
+ accepted, or 207 with accepted/rejected counts and error messages. Do
+ not automatically retry a 207 batch. The generated reference shows only
+ 200; batches return 202 or 207.
operationId: score_create
tags:
- Score
parameters: []
responses:
- '204':
+ '200':
description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/CreateScoresResponse'
'400':
description: ''
content:
@@ -44,9 +52,19 @@ paths:
content:
application/json:
schema:
- $ref: '#/components/schemas/CreateScoreRequest'
+ $ref: '#/components/schemas/CreateScoresRequest'
components:
schemas:
+ CreateScoresRequest:
+ title: CreateScoresRequest
+ oneOf:
+ - $ref: '#/components/schemas/CreateScoreRequest'
+ - $ref: '#/components/schemas/CreateScoreBatchRequest'
+ CreateScoreBatchRequest:
+ title: CreateScoreBatchRequest
+ type: array
+ items:
+ $ref: '#/components/schemas/CreateScoreRequest'
CreateScoreRequest:
title: CreateScoreRequest
type: object
@@ -137,6 +155,56 @@ components:
The value of the score. Must be passed as string for categorical scores,
and numeric for boolean and numeric scores. For corrections pass value
as string and set data type to CORRECTION
+ CreateScoresResponse:
+ title: CreateScoresResponse
+ oneOf:
+ - $ref: '#/components/schemas/CreateScoreResponse'
+ - $ref: '#/components/schemas/CreateScoreBatchResponse'
+ - $ref: '#/components/schemas/CreateScoreBatchResults'
+ CreateScoreBatchResponse:
+ title: CreateScoreBatchResponse
+ type: object
+ description: Batch accepted (HTTP 202).
+ properties:
+ message:
+ type: string
+ const: Accepted
+ required:
+ - message
+ CreateScoreResponse:
+ title: CreateScoreResponse
+ type: object
+ properties:
+ id:
+ type: string
+ description: The id of the created object in Langfuse
+ required:
+ - id
+ CreateScoreBatchResults:
+ title: CreateScoreBatchResults
+ type: object
+ description: Acceptance counts and errors (HTTP 207).
+ properties:
+ accepted:
+ type: integer
+ rejected:
+ type: integer
+ errors:
+ type: array
+ items:
+ $ref: '#/components/schemas/CreateScoreBatchError'
+ required:
+ - accepted
+ - rejected
+ - errors
+ CreateScoreBatchError:
+ title: CreateScoreBatchError
+ type: object
+ properties:
+ message:
+ type: string
+ required:
+ - message
securitySchemes:
BearerAuth:
type: http
diff --git a/web/public/generated/api/openapi.yml b/web/public/generated/api/openapi.yml
index e92700f20c..b5e82d0e68 100644
--- a/web/public/generated/api/openapi.yml
+++ b/web/public/generated/api/openapi.yml
@@ -6842,8 +6842,10 @@ paths:
/api/public/scores:
post:
description: >-
- Create a score (supports trace, observation, session, and dataset run
- scores)
+ Create scores asynchronously. Single score: 200 with ID. Batch: 202 if
+ accepted, or 207 with accepted/rejected counts and error messages. Do
+ not automatically retry a 207 batch. The generated reference shows only
+ 200; batches return 202 or 207.
operationId: scores_create
tags:
- Scores
@@ -6854,7 +6856,7 @@ paths:
content:
application/json:
schema:
- $ref: '#/components/schemas/CreateScoreResponse'
+ $ref: '#/components/schemas/CreateScoresResponse'
'400':
description: ''
content:
@@ -6886,7 +6888,7 @@ paths:
content:
application/json:
schema:
- $ref: '#/components/schemas/CreateScoreRequest'
+ $ref: '#/components/schemas/CreateScoresRequest'
/api/public/v2/scores:
get:
description: >-
@@ -16823,6 +16825,16 @@ components:
required:
- data
- meta
+ CreateScoresRequest:
+ title: CreateScoresRequest
+ oneOf:
+ - $ref: '#/components/schemas/CreateScoreRequest'
+ - $ref: '#/components/schemas/CreateScoreBatchRequest'
+ CreateScoreBatchRequest:
+ title: CreateScoreBatchRequest
+ type: array
+ items:
+ $ref: '#/components/schemas/CreateScoreRequest'
CreateScoreRequest:
title: CreateScoreRequest
type: object
@@ -16907,6 +16919,22 @@ components:
Source values accepted when creating a score via the public REST API.
EVAL is reserved for internal evaluator outputs and is intentionally not
exposed here — use commons.ScoreSource when reading scores.
+ CreateScoresResponse:
+ title: CreateScoresResponse
+ oneOf:
+ - $ref: '#/components/schemas/CreateScoreResponse'
+ - $ref: '#/components/schemas/CreateScoreBatchResponse'
+ - $ref: '#/components/schemas/CreateScoreBatchResults'
+ CreateScoreBatchResponse:
+ title: CreateScoreBatchResponse
+ type: object
+ description: Batch accepted (HTTP 202).
+ properties:
+ message:
+ type: string
+ const: Accepted
+ required:
+ - message
CreateScoreResponse:
title: CreateScoreResponse
type: object
@@ -17062,6 +17090,31 @@ components:
required:
- data
- meta
+ CreateScoreBatchResults:
+ title: CreateScoreBatchResults
+ type: object
+ description: Acceptance counts and errors (HTTP 207).
+ properties:
+ accepted:
+ type: integer
+ rejected:
+ type: integer
+ errors:
+ type: array
+ items:
+ $ref: '#/components/schemas/CreateScoreBatchError'
+ required:
+ - accepted
+ - rejected
+ - errors
+ CreateScoreBatchError:
+ title: CreateScoreBatchError
+ type: object
+ properties:
+ message:
+ type: string
+ required:
+ - message
PaginatedSessions:
title: PaginatedSessions
type: object
diff --git a/web/src/__tests__/server/scores-api-v1.servertest.ts b/web/src/__tests__/server/scores-api-v1.servertest.ts
index f15972c144..d6751c9b32 100644
--- a/web/src/__tests__/server/scores-api-v1.servertest.ts
+++ b/web/src/__tests__/server/scores-api-v1.servertest.ts
@@ -231,6 +231,87 @@ describe("/api/public/scores API Endpoint", () => {
});
describe("POST /api/public/scores", () => {
+ it("should ingest multiple scores posted as a list", async () => {
+ const { projectId, auth } = await createOrgProjectAndApiKey();
+ const traceId = v4();
+ await createTracesCh([
+ createTrace({ id: traceId, project_id: projectId }),
+ ]);
+
+ const scoreIds = [v4(), v4()];
+ const response = await makeAPICall(
+ "POST",
+ "/api/public/scores",
+ scoreIds.map((id, index) => ({
+ id,
+ traceId,
+ name: `batch-score-${index}`,
+ value: index + 1,
+ })),
+ auth,
+ );
+
+ expect(response.status).toBe(202);
+ expect(response.body).toEqual({ message: "Accepted" });
+
+ await waitForExpect(async () => {
+ const scores = await getScoresByIds(projectId, scoreIds);
+ expect(scores).toHaveLength(2);
+ expect(scores).toEqual(
+ expect.arrayContaining(
+ scoreIds.map((id, index) =>
+ expect.objectContaining({
+ id,
+ name: `batch-score-${index}`,
+ value: index + 1,
+ }),
+ ),
+ ),
+ );
+ });
+ });
+
+ it("should summarize accepted and rejected scores in a partially failed batch", async () => {
+ const { auth } = await createOrgProjectAndApiKey();
+ const response = await makeAPICall<{
+ accepted: number;
+ rejected: number;
+ errors: { message: string }[];
+ }>(
+ "POST",
+ "/api/public/scores",
+ [
+ { name: "accepted-score", value: 1, traceId: v4() },
+ { id: "bad\r", name: "rejected-score", value: 1, traceId: v4() },
+ { value: 1, traceId: v4() },
+ {
+ name: "wrong-type",
+ dataType: "NUMERIC",
+ value: "great",
+ traceId: v4(),
+ },
+ null,
+ ],
+ auth,
+ );
+
+ expect(response.status).toBe(207);
+ expect(response.body).toEqual({
+ accepted: 1,
+ rejected: 4,
+ errors: expect.arrayContaining([
+ { message: expect.stringContaining("name") },
+ { message: expect.stringContaining("value") },
+ { message: expect.stringContaining("null") },
+ {
+ message: expect.stringContaining(
+ "ID cannot contain carriage return characters",
+ ),
+ },
+ ]),
+ });
+ });
+
it("should create score for a trace", async () => {
const traceId = v4();
diff --git a/web/src/features/mcp/server/scores/tools/createScore.ts b/web/src/features/mcp/server/scores/tools/createScore.ts
index cbc7ea5398..d1f30c5529 100644
--- a/web/src/features/mcp/server/scores/tools/createScore.ts
+++ b/web/src/features/mcp/server/scores/tools/createScore.ts
@@ -2,7 +2,7 @@ import {
InvalidRequestError,
LangfuseNotFoundError,
PublicApiCreateScoreSourceDomain,
- PostScoresBodyV1,
+ PostScoreBodyV1,
PostScoresResponseV1,
UnauthorizedError,
} from "@langfuse/shared";
@@ -81,7 +81,7 @@ export const [createScoreTool, handleCreateScore] = defineTool({
].join("\n"),
action: "scores:save",
baseSchema: CreateScoreBaseSchema,
- inputSchema: PostScoresBodyV1,
+ inputSchema: PostScoreBodyV1,
destructiveHint: true,
handler: async (input, context) => {
return await runMcpTool({
diff --git a/web/src/features/public-api/server/scores-api-service.ts b/web/src/features/public-api/server/scores-api-service.ts
index 17ef58d23a..e33e05e1b5 100644
--- a/web/src/features/public-api/server/scores-api-service.ts
+++ b/web/src/features/public-api/server/scores-api-service.ts
@@ -11,7 +11,7 @@ import {
InternalServerError,
LISTABLE_SCORE_TYPES,
type ScoreSourceType,
- type PostScoresBodyV1,
+ type PostScoreBodyV1,
scoresTableCols,
type ScoreDataTypeType,
} from "@langfuse/shared";
@@ -227,7 +227,7 @@ export class ScoresApiService {
scoreId = body.id ?? randomUUID(),
attribution,
}: {
- body: z.infer<typeof PostScoresBodyV1>;
+ body: z.infer<typeof PostScoreBodyV1>;
auth: AuthHeaderValidVerificationResultIngestion;
auditScope?: { projectId: string; orgId: string; apiKeyId: string };
scoreId?: string;
@@ -280,6 +280,27 @@ export class ScoresApiService {
return { id: scoreId, result };
}
+ async createScores({
+ bodies,
+ auth,
+ attribution,
+ }: {
+ bodies: unknown[];
+ auth: AuthHeaderValidVerificationResultIngestion;
+ attribution: IngestionAttribution;
+ }) {
+ const events = bodies.map((body) => ({
+ id: randomUUID(),
+ type: eventTypes.SCORE_CREATE,
+ timestamp: new Date().toISOString(),
+ body,
+ }));
+
+ const result = await processEventBatch(events, auth, { attribution });
+
+ return { result };
+ }
+
async deleteScore({
projectId,
orgId,
diff --git a/web/src/pages/api/public/scores/index.ts b/web/src/pages/api/public/scores/index.ts
index 7136c9f958..a43db81f09 100644
--- a/web/src/pages/api/public/scores/index.ts
+++ b/web/src/pages/api/public/scores/index.ts
@@ -30,6 +30,44 @@ export default withMiddlewares({
);
}
+ const scoresApiService = new ScoresApiService("v1");
+ const attribution = createIngestionAttribution({
+ headers: req.headers,
+ authCheck: auth,
+ });
+
+ if (Array.isArray(body)) {
+ const conformedBodies = body.map((score) => {
+ if (
+ typeof score !== "object" ||
+ score === null ||
+ Array.isArray(score)
+ ) {
+ // Leave malformed items for ingestion to reject.
+ return score;
+ }
+ // Generate IDs for scores with missing or empty IDs.
+ return { ...score, id: ("id" in score && score.id) || randomUUID() };
+ });
+ const { result } = await scoresApiService.createScores({
+ bodies: conformedBodies,
+ auth,
+ attribution,
+ });
+ if (result.errors.length > 0) {
+ res.status(207);
+ return {
+ accepted: result.successes.length,
+ rejected: result.errors.length,
+ errors: result.errors.map((error) => ({
+ message: error.error ?? error.message ?? "Failed to create score",
+ })),
+ };
+ }
+ res.status(202);
+ return { message: "Accepted" };
+ }
+
const conformedBody = {
...body,
// We previously used `if(!body.id)` to decide if a new ID should be generated,
@@ -38,14 +76,10 @@ export default withMiddlewares({
id: body.id || randomUUID(),
};
- const scoresApiService = new ScoresApiService("v1");
const { id, result } = await scoresApiService.createScore({
body: conformedBody,
auth,
- attribution: createIngestionAttribution({
- headers: req.headers,
- authCheck: auth,
- }),
+ attribution,
});
if (result.errors.length > 0) {
const error = result.errors[0];