🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
openapi: 3.0.1
info:
title: client
version: ''
paths:
/api/public/scores:
post:
description: >-
Create scores asynchronously. Single score: 200 with ID. Batch: 202 if
accepted, or 207 with accepted/rejected counts and error messages. Do
not automatically retry a 207 batch. The generated reference shows only
200; batches return 202 or 207.
operationId: score_create
tags:
- Score
parameters: []
responses:
'200':
description: ''
content:
application/json:
schema:
$ref: '#/components/schemas/CreateScoresResponse'
'400':
description: ''
content:
application/json:
schema:
type: string
'401':
description: ''
content:
application/json:
schema:
type: string
'403':
description: ''
content:
application/json:
schema:
type: string
'405':
description: ''
content:
application/json:
schema:
type: string
security:
- BearerAuth: []
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/CreateScoresRequest'
components:
schemas:
CreateScoresRequest:
title: CreateScoresRequest
oneOf:
- $ref: '#/components/schemas/CreateScoreRequest'
- $ref: '#/components/schemas/CreateScoreBatchRequest'
CreateScoreBatchRequest:
title: CreateScoreBatchRequest
type: array
items:
$ref: '#/components/schemas/CreateScoreRequest'
CreateScoreRequest:
title: CreateScoreRequest
type: object
properties:
id:
type: string
nullable: true
traceId:
type: string
nullable: true
sessionId:
type: string
nullable: true
observationId:
type: string
nullable: true
datasetRunId:
type: string
nullable: true
name:
type: string
description: >-
The name of the score. Always overrides "output" for correction
scores.
value:
$ref: '#/components/schemas/CreateScoreValue'
description: >-
The value of the score. Must be passed as string for categorical
scores, and numeric for boolean and numeric scores. Boolean score
values must equal either 1 or 0 (true or false)
comment:
type: string
nullable: true
metadata:
nullable: true
dataType:
$ref: '#/components/schemas/ScoreDataType'
nullable: true
description: >-
When set, must match the score value's type. If not set, will be
inferred from the score value or config
configId:
type: string
nullable: true
description: >-
Reference a score config on a score. When set, the score name must
equal the config name and scores must comply with the config's range
and data type. For categorical scores, the value must map to a
config category. Numeric scores might be constrained by the score
config's max and min values
source:
$ref: '#/components/schemas/CreateScoreSource'
nullable: true
description: >-
The source of the score. Defaults to API. Set to ANNOTATION to
prefill scores (e.g. from an LLM) for a human reviewer to verify in
an annotation queue. When source is ANNOTATION, a configId is
required unless dataType is CORRECTION. EVAL is reserved for
internal evaluator outputs and is not accepted on this endpoint.
required:
- name
- value
CreateScoreSource:
title: CreateScoreSource
type: string
enum:
- API
- ANNOTATION
description: |-
Source values accepted when creating a score via the public API.
EVAL is reserved for internal evaluator outputs and is intentionally not
exposed here.
ScoreDataType:
title: ScoreDataType
type: string
enum:
- NUMERIC
- CATEGORICAL
- BOOLEAN
- CORRECTION
CreateScoreValue:
title: CreateScoreValue
oneOf:
- type: number
format: double
- type: string
description: >-
The value of the score. Must be passed as string for categorical scores,
and numeric for boolean and numeric scores. For corrections pass value
as string and set data type to CORRECTION
CreateScoresResponse:
title: CreateScoresResponse
oneOf:
- $ref: '#/components/schemas/CreateScoreBatchResults'
- $ref: '#/components/schemas/CreateScoreResponse'
- $ref: '#/components/schemas/CreateScoreBatchResponse'
CreateScoreBatchResponse:
title: CreateScoreBatchResponse
type: object
description: Batch accepted (HTTP 202).
properties:
message:
type: string
const: Accepted
required:
- message
CreateScoreResponse:
title: CreateScoreResponse
type: object
properties:
id:
type: string
description: The id of the created object in Langfuse
required:
- id
CreateScoreBatchResults:
title: CreateScoreBatchResults
type: object
description: Acceptance counts and errors (HTTP 207).
properties:
accepted:
type: integer
rejected:
type: integer
errors:
type: array
items:
$ref: '#/components/schemas/CreateScoreBatchError'
required:
- accepted
- rejected
- errors
CreateScoreBatchError:
title: CreateScoreBatchError
type: object
properties:
message:
type: string
required:
- message
securitySchemes:
BearerAuth:
type: http
scheme: bearer