Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(api): add stable evaluator and evaluation rule endpoints (#16608)

Tobias Wochinger96d0ce458689
diff --git a/fern/apis/server/definition/evaluation-commons.yml b/fern/apis/server/definition/evaluation-commons.yml
new file mode 100644
index 0000000000..2c43010194
--- /dev/null
+++ b/fern/apis/server/definition/evaluation-commons.yml
@@ -0,0 +1,606 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
+types:
+ EvaluatorType:
+ docs: |
+ The evaluator type.
+
+ The public API supports LLM-as-a-judge and code evaluators.
+ enum:
+ - llm_as_judge
+ - code
+
+ CodeEvaluatorSourceCodeLanguage:
+ docs: Code evaluator runtime language.
+ enum:
+ - PYTHON
+ - TYPESCRIPT
+
+ PromptVariableMappingSource:
+ docs: |
+ Source field used to populate a prompt variable.
+
+ Use these values when mapping evaluator prompt variables to live data.
+
+ Source semantics:
+ - `input`: the observation input payload
+ - `output`: the observation output payload
+ - `metadata`: the observation metadata object. Combine with `jsonPath` when you need one nested field instead of the whole object.
+ - `tool_calls`: the tool calls recorded on the observation, as an array of `{id, name, arguments, type, index}` objects in the order the model emitted them. Combine with `jsonPath` (for example `$[*].name`) to select parts of each call.
+ - `expected_output`: the experiment item's expected output when the observation belongs to an experiment.
+ - `experiment_item_metadata`: the experiment item's metadata when the observation belongs to an experiment.
+ enum:
+ - input
+ - output
+ - metadata
+ - tool_calls
+ - expected_output
+ - experiment_item_metadata
+
+ EvaluatorOutputScoreType:
+ docs: |
+ Structured score type returned by an evaluator.
+
+ This controls the type of score value Langfuse stores for evaluation results:
+ - `NUMERIC`: a numeric score such as `0.82`
+ - `BOOLEAN`: a boolean score such as `true`
+ - `CATEGORICAL`: one or more category labels from a fixed list
+ enum:
+ - NUMERIC
+ - BOOLEAN
+ - CATEGORICAL
+
+ EvaluatorOutputDefinition:
+ docs: |
+ Flat structured output definition used when creating or updating an evaluator.
+
+ - `dataType` is required.
+ - `scoreReasoningInstructions` and `scoreValueInstructions` are optional instructions.
+ - `minValue` and `maxValue` apply only to `NUMERIC` outputs. If both are set, `minValue` must not exceed `maxValue`.
+ - `categories` and `shouldAllowMultipleMatches` apply only to `CATEGORICAL` outputs.
+ - Do not send `version`; that is an internal storage detail.
+ discriminant: dataType
+ union:
+ NUMERIC:
+ type: PublicEvaluatorNumericScore
+ BOOLEAN:
+ type: PublicEvaluatorBooleanScore
+ CATEGORICAL:
+ type: PublicEvaluatorCategoricalScore
+ examples:
+ - name: Numeric
+ value:
+ dataType: NUMERIC
+ minValue: 0
+ maxValue: 1
+ - name: Boolean
+ value:
+ dataType: BOOLEAN
+ scoreValueInstructions: Return true if the output satisfies the requirement, otherwise false.
+ - name: Categorical
+ value:
+ dataType: CATEGORICAL
+ scoreReasoningInstructions: Explain which category best fits the output.
+ scoreValueInstructions: Choose the best category.
+ categories:
+ - correct
+ - partially_correct
+ - incorrect
+ shouldAllowMultipleMatches: false
+
+ EvaluatorOutputDefinitionBase:
+ properties:
+ scoreReasoningInstructions:
+ type: optional<string>
+ docs: Optional instructions for deriving the reasoning returned with the score.
+ scoreValueInstructions:
+ type: optional<string>
+ docs: Optional instructions for deriving the score value.
+
+ PublicEvaluatorNumericScore:
+ extends: EvaluatorOutputDefinitionBase
+ properties:
+ dataType:
+ type: literal<"NUMERIC">
+ docs: Numeric score output.
+ minValue:
+ type: optional<double>
+ docs: Optional inclusive minimum value.
+ maxValue:
+ type: optional<double>
+ docs: Optional inclusive maximum value.
+
+ PublicEvaluatorBooleanScore:
+ extends: EvaluatorOutputDefinitionBase
+ properties:
+ dataType:
+ type: literal<"BOOLEAN">
+ docs: Boolean score output.
+
+ PublicEvaluatorCategoricalScore:
+ extends: EvaluatorOutputDefinitionBase
+ properties:
+ dataType:
+ type: literal<"CATEGORICAL">
+ docs: Categorical score output.
+ categories:
+ type: list<string>
+ docs: Allowed category values. At least two unique values are required.
+ shouldAllowMultipleMatches:
+ type: boolean
+ docs: Whether the evaluator may return more than one category.
+
+ PublicEvaluatorOutputDefinition:
+ docs: |
+ Flat evaluator output definition returned by the public API.
+
+ This response always includes `dataType` and never includes an internal output-definition `version`.
+ Optional empty descriptions from legacy definitions are omitted.
+ discriminant: dataType
+ union:
+ NUMERIC:
+ type: PublicEvaluatorNumericScore
+ BOOLEAN:
+ type: PublicEvaluatorBooleanScore
+ CATEGORICAL:
+ type: PublicEvaluatorCategoricalScore
+ examples:
+ - name: PublicNumeric
+ value:
+ dataType: NUMERIC
+ minValue: 0
+ maxValue: 1
+ - name: PublicCategorical
+ value:
+ dataType: CATEGORICAL
+ scoreReasoningInstructions: Explain which label best fits the output.
+ scoreValueInstructions: Choose the best label.
+ categories:
+ - correct
+ - partially_correct
+ - incorrect
+ shouldAllowMultipleMatches: false
+
+ EvaluationRuleStringFilterOperator:
+ enum:
+ - name: Equals
+ value: "="
+ - name: Contains
+ value: contains
+ - name: DoesNotContain
+ value: does not contain
+ - name: StartsWith
+ value: starts with
+ - name: EndsWith
+ value: ends with
+
+ EvaluationRuleNumberFilterOperator:
+ enum:
+ - name: Equals
+ value: "="
+ - name: GreaterThan
+ value: ">"
+ - name: LessThan
+ value: "<"
+ - name: GreaterThanOrEqual
+ value: ">="
+ - name: LessThanOrEqual
+ value: "<="
+
+ EvaluationRuleOptionsFilterOperator:
+ enum:
+ - name: AnyOf
+ value: any of
+ - name: NoneOf
+ value: none of
+
+ EvaluationRuleArrayOptionsFilterOperator:
+ enum:
+ - name: AnyOf
+ value: any of
+ - name: NoneOf
+ value: none of
+ - name: AllOf
+ value: all of
+
+ EvaluationRuleBooleanFilterOperator:
+ enum:
+ - name: Equals
+ value: "="
+ - name: NotEquals
+ value: "<>"
+
+ EvaluationRuleNullFilterOperator:
+ enum:
+ - name: IsNull
+ value: is null
+ - name: IsNotNull
+ value: is not null
+
+ DateTimeEvaluationRuleFilter:
+ properties:
+ column:
+ type: string
+ docs: Column to filter on.
+ operator:
+ type: EvaluationRuleNumberFilterOperator
+ docs: Comparison operator for datetime values.
+ value:
+ type: datetime
+ docs: Datetime value to compare against.
+
+ StringEvaluationRuleFilter:
+ properties:
+ column:
+ type: string
+ docs: Column to filter on.
+ operator:
+ type: EvaluationRuleStringFilterOperator
+ value:
+ type: string
+
+ NumberEvaluationRuleFilter:
+ properties:
+ column:
+ type: string
+ docs: Column to filter on.
+ operator:
+ type: EvaluationRuleNumberFilterOperator
+ value:
+ type: double
+
+ StringOptionsEvaluationRuleFilter:
+ properties:
+ column:
+ type: string
+ docs: Column to filter on.
+ operator:
+ type: EvaluationRuleOptionsFilterOperator
+ value:
+ type: list<string>
+ docs: One or more allowed string values.
+
+ ArrayOptionsEvaluationRuleFilter:
+ properties:
+ column:
+ type: string
+ docs: Column to filter on.
+ operator:
+ type: EvaluationRuleArrayOptionsFilterOperator
+ value:
+ type: list<string>
+ docs: One or more array elements to match.
+
+ StringObjectEvaluationRuleFilter:
+ properties:
+ column:
+ type: string
+ docs: Object-valued column to filter on. Currently only `metadata` is supported.
+ key:
+ type: string
+ docs: Top-level key inside the object-valued column to filter on.
+ operator:
+ type: EvaluationRuleStringFilterOperator
+ value:
+ type: string
+
+ NumberObjectEvaluationRuleFilter:
+ properties:
+ column:
+ type: string
+ docs: Object-valued column to filter on.
+ key:
+ type: string
+ docs: Key inside the object-valued column to filter on.
+ operator:
+ type: EvaluationRuleNumberFilterOperator
+ value:
+ type: double
+
+ CategoryOptionsEvaluationRuleFilter:
+ properties:
+ column:
+ type: string
+ docs: Object-valued column to filter on.
+ key:
+ type: string
+ docs: Key inside the object-valued column to filter on.
+ operator:
+ type: EvaluationRuleOptionsFilterOperator
+ value:
+ type: list<string>
+
+ BooleanEvaluationRuleFilter:
+ properties:
+ column:
+ type: string
+ docs: Column to filter on.
+ operator:
+ type: EvaluationRuleBooleanFilterOperator
+ value:
+ type: boolean
+
+ NullEvaluationRuleFilter:
+ properties:
+ column:
+ type: string
+ docs: Column to filter on, for example `parentObservationId`.
+ operator:
+ type: EvaluationRuleNullFilterOperator
+ value:
+ type: literal<"">
+ docs: Required empty-string placeholder used by the runtime filter contract.
+
+ PromptVariableMappingInput:
+ docs: |
+ Connects one prompt variable to data from an observation or experiment.
+
+ Manual mappings are used for `llm_as_judge` evaluators. `code` evaluators use a fixed runtime mapping managed by Langfuse.
+
+ How to build a valid mapping list:
+ 1. Create the evaluator or fetch it with `GET /evaluators/{id}`.
+ 2. Read the evaluator `variables` array.
+ 3. Add exactly one mapping object for each variable in that array.
+ 4. Use the variable name exactly as returned, without braces such as `{{` or `}}`.
+ 5. Choose the source field that should populate the variable.
+
+ `jsonPath` is optional. Use it only when the selected source is a JSON object and you want to extract one nested field before inserting it into the evaluator prompt.
+
+ Invalid, missing, or duplicate mappings return a validation error. Malformed JSONPath expressions are also rejected.
+ properties:
+ variable:
+ type: string
+ docs: |
+ Prompt variable name without braces.
+
+ Example: for the prompt `Judge {{input}} against {{output}}`, use `input` and `output`.
+ source:
+ type: PromptVariableMappingSource
+ docs: |
+ Source field that should populate the prompt variable.
+
+ Available sources are `input`, `output`, `metadata`, `tool_calls`, `expected_output`, and `experiment_item_metadata`.
+ jsonPath:
+ type: optional<string>
+ docs: |
+ Optional JSONPath selector applied to the selected source before it is passed to the evaluator prompt.
+
+ Requirements:
+ - Must start with `$`
+ - Must be a syntactically valid JSONPath expression
+ - Most useful with `source=metadata`
+ examples:
+ - name: BasicObservationMapping
+ value:
+ variable: input
+ source: input
+ - name: MetadataProjectionMapping
+ value:
+ variable: customer_tier
+ source: metadata
+ jsonPath: "$.customer.tier"
+ - name: ExperimentExpectedOutputMapping
+ value:
+ variable: expected_output
+ source: expected_output
+
+ PromptVariableMappingRead:
+ docs: |
+ Connects one prompt variable to source data.
+
+ `source` is `null` when the mapping is incomplete.
+ properties:
+ variable:
+ type: string
+ docs: Prompt variable name without braces.
+ source:
+ type: nullable<string>
+ docs: Stored source field populating the variable, or `null` when the mapping is incomplete.
+ jsonPath:
+ type: optional<string>
+ docs: Optional JSONPath selector applied to the selected source.
+ examples:
+ - name: ConfiguredReadMapping
+ value:
+ variable: input
+ source: input
+ - name: IncompleteReadMapping
+ value:
+ variable: input
+ source: null
+
+ PromptVariableMapping:
+ docs: Connects one prompt variable to source data.
+ discriminated: false
+ union:
+ - PromptVariableMappingRead
+ - LegacyPromptVariableMapping
+
+ LegacyPromptVariableMapping:
+ docs: |
+ **Deprecated:** Connects one prompt variable to data from a legacy trace or dataset evaluation rule.
+
+ `langfuseObject` selects the object kind. `objectName` separately selects a named observation and is `null` for trace and dataset-item mappings.
+ properties:
+ mappingType:
+ type: literal<"legacy">
+ docs: Explicitly marks this as a legacy mapping.
+ variable:
+ type: string
+ docs: Prompt variable name without braces.
+ langfuseObject:
+ type: LegacyEvaluationObject
+ docs: Legacy object kind selected as the mapping source.
+ objectName:
+ type: nullable<string>
+ docs: Observation name to match, or `null` when `langfuseObject` is `trace` or `dataset_item`.
+ source:
+ type: string
+ docs: Field selected from the legacy object.
+ jsonPath:
+ type: optional<string>
+ docs: Optional JSONPath selector applied to the selected field.
+ examples:
+ - name: LegacyTraceInput
+ value:
+ mappingType: legacy
+ variable: input
+ langfuseObject: trace
+ objectName: null
+ source: input
+ - name: LegacyNamedGenerationOutput
+ value:
+ mappingType: legacy
+ variable: output
+ langfuseObject: generation
+ objectName: answer-generation
+ source: output
+ jsonPath: "$.answer"
+
+ LegacyEvaluationObject:
+ docs: "**Deprecated:** Legacy Langfuse object kind used by trace and dataset evaluation rules."
+ enum:
+ - trace
+ - span
+ - generation
+ - event
+ - agent
+ - tool
+ - chain
+ - retriever
+ - evaluator
+ - embedding
+ - guardrail
+ - dataset_item
+
+ EvaluationRuleFilter:
+ docs: |
+ One filter condition used to decide whether a live-ingested observation should be evaluated.
+
+ All filters must be satisfied for the rule to run. Experiment scope is expressed with filters rather than a separate target field:
+ - `isExperimentItemRootSpan = true` limits execution to experiment item roots.
+ - `datasetId` limits execution to experiments for the selected datasets. Use dataset IDs from `GET /api/public/v2/datasets`.
+
+ Pick the filter `type` first, as it determines the required fields and value shape. Use `key` only for object filters such as `metadata`.
+
+ Operator quick reference by filter `type`:
+ - `string`: `"="`, `contains`, `does not contain`, `starts with`, `ends with`
+ - `number`: `"="`, `">"`, `"<"`, `">="`, `"<="`
+ - `datetime`: `"="`, `">"`, `"<"`, `">="`, `"<="`
+ - `stringOptions`: `any of`, `none of`
+ - `arrayOptions`: `any of`, `none of`, `all of`
+ - `stringObject`: same operators as `string`
+ - `boolean`: `"="`, `"<>"`
+ - `null`: `is null`, `is not null`
+
+ Supported columns. Each column accepts exactly one filter `type`; sending a column with a different `type` is rejected with `400`.
+ - `type`: `stringOptions`, values `SPAN`, `EVENT`, `GENERATION`, `AGENT`, `TOOL`, `CHAIN`, `RETRIEVER`, `EVALUATOR`, `EMBEDDING`, `GUARDRAIL`
+ - `name`: `stringOptions`
+ - `environment`: `stringOptions`
+ - `level`: `stringOptions`, values `DEBUG`, `DEFAULT`, `WARNING`, `ERROR`
+ - `version`: `string`
+ - `traceName`: `stringOptions`
+ - `userId`: `string`
+ - `sessionId`: `string`
+ - `tags`: `arrayOptions`
+ - `metadata`: `stringObject`, requires `key`
+ - `isRootObservation`: `boolean`; true when the observation has no parent or is explicitly marked as an application root
+ - `parentObservationId`: `null`
+ - `experimentId`: `stringOptions`
+ - `isExperimentItemRootSpan`: `boolean`
+ - `calledToolNames`: `arrayOptions`
+ - `toolCalls`: `number`, the number of tool calls on the observation
+ - `datasetId`: `stringOptions`. Use dataset `id` values from `GET /api/public/v2/datasets`, not dataset names.
+ discriminant: type
+ union:
+ "datetime":
+ type: DateTimeEvaluationRuleFilter
+ "string":
+ type: StringEvaluationRuleFilter
+ "number":
+ type: NumberEvaluationRuleFilter
+ "stringOptions":
+ type: StringOptionsEvaluationRuleFilter
+ "categoryOptions":
+ type: CategoryOptionsEvaluationRuleFilter
+ "arrayOptions":
+ type: ArrayOptionsEvaluationRuleFilter
+ "stringObject":
+ type: StringObjectEvaluationRuleFilter
+ "numberObject":
+ type: NumberObjectEvaluationRuleFilter
+ "boolean":
+ type: BooleanEvaluationRuleFilter
+ "null":
+ type: NullEvaluationRuleFilter
+ examples:
+ - name: ObservationTypeFilter
+ value:
+ type: stringOptions
+ column: type
+ operator: any of
+ value:
+ - GENERATION
+ - name: ObservationMetadataFilter
+ value:
+ type: stringObject
+ column: metadata
+ key: customerTier
+ operator: "="
+ value: enterprise
+ - name: ObservationRootOnlyFilter
+ value:
+ type: "null"
+ column: parentObservationId
+ operator: is null
+ value: ""
+ - name: ObservationLogicalRootFilter
+ value:
+ type: boolean
+ column: isRootObservation
+ operator: "="
+ value: true
+ - name: ObservationTagsFilter
+ value:
+ type: arrayOptions
+ column: tags
+ operator: any of
+ value:
+ - production
+ - name: ExperimentDatasetFilter
+ value:
+ type: stringOptions
+ column: datasetId
+ operator: any of
+ value:
+ - "550e8400-e29b-41d4-a716-446655440000"
+
+ EvaluationRuleReadFilterBase:
+ docs: Stored filter returned verbatim by evaluation-rule read endpoints.
+ properties:
+ type:
+ type: string
+ docs: Stored filter type. This is not broken down into separate public response types.
+ column:
+ type: string
+ operator:
+ type: string
+ value:
+ type: optional<unknown>
+ docs: Stored filter value. Its shape depends on the filter type.
+
+ EvaluationRuleReadFilterWithKey:
+ extends: EvaluationRuleReadFilterBase
+ properties:
+ key:
+ type: string
+ docs: Stored object or legacy filter key.
+
+ EvaluationRuleReadFilter:
+ docs: |
+ Filter returned verbatim by evaluation-rule read endpoints.
+
+ Filters with a stored `key` use the keyed shape. All other filters use the base shape. Response filters are intentionally not broken down by their internal `type` value.
+ discriminated: false
+ union:
+ - EvaluationRuleReadFilterWithKey
+ - EvaluationRuleReadFilterBase
diff --git a/fern/apis/server/definition/evaluation-errors.yml b/fern/apis/server/definition/evaluation-errors.yml
new file mode 100644
index 0000000000..0e514da84b
--- /dev/null
+++ b/fern/apis/server/definition/evaluation-errors.yml
@@ -0,0 +1,125 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
+types:
+ PublicApiErrorCode:
+ docs: Stable machine-readable error code returned by the evaluators and evaluation-rules API.
+ enum:
+ - authentication_failed
+ - access_denied
+ - invalid_request
+ - invalid_query
+ - invalid_body
+ - resource_not_found
+ - conflict
+ - rate_limited
+ - method_not_allowed
+ - internal_error
+
+ PublicApiValidationIssue:
+ docs: One request validation issue.
+ properties:
+ code:
+ type: string
+ docs: Validator issue code.
+ message:
+ type: string
+ docs: Human-readable explanation.
+ path:
+ type: list<unknown>
+ docs: Path to the invalid request field.
+
+ PublicApiErrorDetails:
+ docs: Optional structured context for validation and rate-limit errors.
+ properties:
+ issues:
+ type: optional<list<PublicApiValidationIssue>>
+ docs: Validation issues for an invalid request body or query.
+ retryAfterSeconds:
+ type: optional<integer>
+ docs: Number of seconds to wait before retrying a rate-limited request.
+ limit:
+ type: optional<integer>
+ docs: Rate-limit request allowance.
+ remaining:
+ type: optional<integer>
+ docs: Remaining requests in the current rate-limit window.
+ resetAt:
+ type: optional<datetime>
+ docs: Time when the current rate-limit window resets.
+
+ PublicApiError:
+ docs: |
+ Standard error response for the stable evaluators and evaluation-rules API.
+
+ Use the HTTP status for the broad failure class and `code` for programmatic handling. `details` is included when field-level validation or retry information is available.
+ properties:
+ message:
+ type: string
+ docs: Human-readable description of the failure.
+ code:
+ type: PublicApiErrorCode
+ docs: Stable machine-readable error code.
+ details:
+ type: optional<PublicApiErrorDetails>
+ docs: Optional structured validation or rate-limit context.
+ examples:
+ - name: InvalidBody
+ value:
+ message: Invalid request body
+ code: invalid_body
+ details:
+ issues:
+ - code: invalid_type
+ message: Invalid input
+ path:
+ - definition
+ - name: ResourceNotFound
+ value:
+ message: Evaluator not found
+ code: resource_not_found
+ - name: RateLimited
+ value:
+ message: Rate limit exceeded
+ code: rate_limited
+ details:
+ retryAfterSeconds: 60
+ limit: 1000
+ remaining: 0
+ resetAt: "2026-03-30T10:00:00.000Z"
+
+errors:
+ BadRequestError:
+ docs: Request validation failed or the requested state is invalid.
+ status-code: 400
+ type: PublicApiError
+ UnauthorizedError:
+ docs: Authentication failed.
+ status-code: 401
+ type: PublicApiError
+ AccessDeniedError:
+ docs: The caller is not allowed to access this resource.
+ status-code: 403
+ type: PublicApiError
+ NotFoundError:
+ docs: The requested evaluator or evaluation rule was not found in the authenticated project.
+ status-code: 404
+ type: PublicApiError
+ MethodNotAllowedError:
+ docs: The HTTP method is not supported by this endpoint.
+ status-code: 405
+ type: PublicApiError
+ ConflictError:
+ docs: The request conflicts with the current evaluator or evaluation-rule state.
+ status-code: 409
+ type: PublicApiError
+ PreconditionFailedError:
+ docs: The evaluator configuration cannot be used in its current form.
+ status-code: 412
+ type: PublicApiError
+ TooManyRequestsError:
+ docs: The project is rate limited. Use the response details to decide when to retry.
+ status-code: 429
+ type: PublicApiError
+ InternalServerError:
+ docs: An unexpected server-side error occurred.
+ status-code: 500
+ type: PublicApiError
diff --git a/fern/apis/server/definition/evaluation-rules.yml b/fern/apis/server/definition/evaluation-rules.yml
new file mode 100644
index 0000000000..9f3853de3d
--- /dev/null
+++ b/fern/apis/server/definition/evaluation-rules.yml
@@ -0,0 +1,241 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
+imports:
+ evals: ./evaluation-commons.yml
+ evaluators: ./evaluators.yml
+ errors: ./evaluation-errors.yml
+
+service:
+ auth: true
+ base-path: /api/public/v2/evaluation-rules
+ endpoints:
+ create:
+ docs: |
+ Create an evaluation rule using stable evaluator identifiers.
+
+ An evaluation rule defines **which** incoming observations should be evaluated and how prompt variables should be populated.
+
+ Key behavior:
+ - rule names are not identifiers and do not need to be unique
+ - rules always use the latest version of each associated evaluator
+ - an enabled rule requires at least one evaluator assignment
+ - omit `sampling` to evaluate every matching observation
+ - omit `filter` to match every incoming observation
+ - `datasetId` is the public filter name for selecting experiment datasets
+ method: POST
+ path: ""
+ request: CreateEvaluationRuleRequest
+ response:
+ status-code: 201
+ type: EvaluationRule
+ errors:
+ - errors.BadRequestError
+ - errors.UnauthorizedError
+ - errors.AccessDeniedError
+ - errors.NotFoundError
+ - errors.MethodNotAllowedError
+ - errors.ConflictError
+ - errors.TooManyRequestsError
+ - errors.InternalServerError
+
+ list:
+ docs: |
+ List evaluation rules in newest-first creation order.
+
+ This includes legacy trace and dataset rules so they can be inspected and migrated. Treat the cursor as opaque and return it unchanged.
+ method: GET
+ path: ""
+ request:
+ name: ListEvaluationRulesRequest
+ query-parameters:
+ limit:
+ type: optional<integer>
+ docs: Maximum number of items to return. Defaults to `50` and cannot exceed `100`.
+ cursor:
+ type: optional<string>
+ docs: Opaque cursor returned by the previous page.
+ response: EvaluationRulesPage
+ errors:
+ - errors.BadRequestError
+ - errors.UnauthorizedError
+ - errors.AccessDeniedError
+ - errors.MethodNotAllowedError
+ - errors.TooManyRequestsError
+ - errors.InternalServerError
+
+ get:
+ docs: Get one evaluation rule, including a legacy trace or dataset rule, by its stable identifier.
+ method: GET
+ path: /{evaluationRuleId}
+ path-parameters:
+ evaluationRuleId:
+ type: string
+ docs: Stable evaluation-rule identifier returned by the evaluation-rule endpoints.
+ response: EvaluationRule
+ errors:
+ - errors.UnauthorizedError
+ - errors.AccessDeniedError
+ - errors.NotFoundError
+ - errors.MethodNotAllowedError
+ - errors.TooManyRequestsError
+ - errors.InternalServerError
+
+ update:
+ docs: |
+ Update an evaluation rule by its stable identifier.
+
+ Provide only the fields to change. Providing `evaluatorAssignments` replaces the complete assignment list. Replacing the list with an empty array disables the rule. Setting `enabled=true` is rejected when the resulting assignment list is empty, including when both fields are sent in the same request.
+
+ Legacy trace and dataset rules follow the existing lifecycle restrictions: they can be deactivated with `enabled=false`, but their name, filters, sampling, and evaluator assignments cannot be changed.
+ method: PATCH
+ path: /{evaluationRuleId}
+ path-parameters:
+ evaluationRuleId:
+ type: string
+ docs: Stable evaluation-rule identifier returned by the evaluation-rule endpoints.
+ request: UpdateEvaluationRuleRequest
+ response: EvaluationRule
+ errors:
+ - errors.BadRequestError
+ - errors.UnauthorizedError
+ - errors.AccessDeniedError
+ - errors.NotFoundError
+ - errors.MethodNotAllowedError
+ - errors.ConflictError
+ - errors.TooManyRequestsError
+ - errors.InternalServerError
+
+ delete:
+ docs: |
+ Delete an evaluation rule.
+
+ This removes the live-ingestion rule only. It does not delete associated evaluators or scores already produced by them.
+
+ Legacy trace and dataset rules can also be deleted. Their evaluators and previously produced scores are preserved.
+ method: DELETE
+ path: /{evaluationRuleId}
+ path-parameters:
+ evaluationRuleId:
+ type: string
+ docs: Stable evaluation-rule identifier returned by the evaluation-rule endpoints.
+ response: DeletedEvaluationRule
+ errors:
+ - errors.UnauthorizedError
+ - errors.AccessDeniedError
+ - errors.NotFoundError
+ - errors.MethodNotAllowedError
+ - errors.TooManyRequestsError
+ - errors.InternalServerError
+
+types:
+ EvaluationRuleEvaluatorAssignmentInput:
+ properties:
+ evaluatorId:
+ type: string
+ docs: Stable evaluator identifier. The rule automatically uses that evaluator's latest version.
+ variableMapping:
+ type: optional<nullable<list<evals.PromptVariableMappingInput>>>
+ docs: |
+ Rule-specific prompt-variable mapping.
+
+ Set to `null` or omit to inherit the evaluator's latest default mapping. Code evaluators use the fixed runtime mapping and should use `null`.
+
+ EvaluatorAssignment:
+ properties:
+ evaluatorId:
+ type: string
+ docs: Stable identifier of the evaluator associated with this rule.
+ variableMapping:
+ type: nullable<list<evals.PromptVariableMapping>>
+ docs: Stored rule-specific override, or `null` when the evaluator's latest default mapping is inherited. Legacy mappings are returned as an explicit `legacy` variant.
+
+ EvaluationRule:
+ docs: |
+ Live evaluation rule for incoming observations.
+
+ A rule determines which evaluators should be used, which observations should trigger scoring, how often scoring should run, and which observation fields should populate prompt variables.
+ properties:
+ id:
+ type: string
+ docs: Stable evaluation-rule identifier.
+ name:
+ type: string
+ docs: Human-readable rule name. This is independent from evaluator names and does not need to be unique.
+ createdBy:
+ type: nullable<evaluators.Creator>
+ docs: User who created this rule, or `null` when no user can be resolved.
+ enabled:
+ type: boolean
+ docs: Whether live execution is enabled for this rule.
+ sampling:
+ type: double
+ docs: |
+ Fraction of matching observations that should be evaluated.
+
+ Must be between `0` and `1`.
+ - `1` evaluates every matching observation.
+ - `0.25` evaluates approximately 25% of matching observations.
+ filter:
+ type: list<evals.EvaluationRuleReadFilter>
+ docs: List of stored filter conditions returned verbatim. Filters with a `key` use the keyed response shape; all others use the base shape. These response shapes are not broken down by internal filter type. An empty list matches every incoming object.
+ evaluatorAssignments:
+ type: list<EvaluatorAssignment>
+ docs: Evaluators attached to this rule in deterministic assignment order.
+ createdAt:
+ type: datetime
+ docs: Timestamp when the evaluation rule was created.
+ updatedAt:
+ type: datetime
+ docs: Timestamp when the evaluation rule was last updated.
+
+ CreateEvaluationRuleRequest:
+ properties:
+ name:
+ type: string
+ docs: Human-readable rule name. Names are not identifiers and do not need to be unique.
+ enabled:
+ type: boolean
+ docs: Whether live execution should start immediately. Enabled rules require at least one evaluator assignment.
+ sampling:
+ type: optional<double>
+ docs: Fraction of matching observations to evaluate. Omit this field to use the default of `1`, which evaluates every match. `null` is not accepted.
+ filter:
+ type: optional<list<evals.EvaluationRuleFilter>>
+ docs: Conditions used to select observations. Defaults to an empty list, which matches every incoming observation.
+ evaluatorAssignments:
+ type: list<EvaluationRuleEvaluatorAssignmentInput>
+ docs: Evaluators to attach to this rule. Disabled rules may use an empty list as a draft.
+
+ UpdateEvaluationRuleRequest:
+ docs: At least one field must be provided.
+ properties:
+ name:
+ type: optional<string>
+ docs: New human-readable rule name.
+ enabled:
+ type: optional<boolean>
+ docs: New desired live-execution state.
+ sampling:
+ type: optional<double>
+ docs: New fraction of matching observations to evaluate. Omit to keep the current value.
+ filter:
+ type: optional<list<evals.EvaluationRuleFilter>>
+ docs: Complete replacement filter list. An empty list matches every incoming observation.
+ evaluatorAssignments:
+ type: optional<list<EvaluationRuleEvaluatorAssignmentInput>>
+ docs: Complete replacement assignment list. An empty list disables the rule.
+
+ EvaluationRulesPage:
+ properties:
+ data:
+ type: list<EvaluationRule>
+ docs: Evaluation rules for this page.
+ meta:
+ type: evaluators.CursorMeta
+ docs: Cursor pagination metadata.
+
+ DeletedEvaluationRule:
+ docs: Confirmation returned after successful evaluation-rule deletion.
+ properties:
+ id:
+ type: string
+ docs: Identifier of the deleted evaluation rule.
diff --git a/fern/apis/server/definition/evaluators.yml b/fern/apis/server/definition/evaluators.yml
new file mode 100644
index 0000000000..45778b3fa1
--- /dev/null
+++ b/fern/apis/server/definition/evaluators.yml
@@ -0,0 +1,529 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
+imports:
+ evals: ./evaluation-commons.yml
+ errors: ./evaluation-errors.yml
+
+service:
+ auth: true
+ base-path: /api/public/v2/evaluators
+ endpoints:
+ create:
+ docs: |
+ Create an evaluator in the authenticated project.
+
+ An evaluator defines **how** Langfuse should score data. LLM-as-a-judge evaluators define a prompt, expected structured output, and optional model configuration. Code evaluators define source code and a runtime language.
+
+ This always creates a new evaluator with version `1`. Names are not identifiers and do not need to be unique.
+ method: POST
+ path: ""
+ request: CreateEvaluatorRequest
+ response:
+ status-code: 201
+ type: Evaluator
+ errors:
+ - errors.BadRequestError
+ - errors.UnauthorizedError
+ - errors.AccessDeniedError
+ - errors.MethodNotAllowedError
+ - errors.PreconditionFailedError
+ - errors.TooManyRequestsError
+ - errors.InternalServerError
+
+ list:
+ docs: |
+ List evaluators in newest-first creation order.
+
+ Every evaluator includes its latest definition and version metadata flattened into the evaluator object, plus associated evaluation rules. Treat the cursor as opaque and return it unchanged.
+ method: GET
+ path: ""
+ request:
+ name: ListEvaluatorsRequest
+ query-parameters:
+ limit:
+ type: optional<integer>
+ docs: Maximum number of items to return. Defaults to `50` and cannot exceed `100`.
+ cursor:
+ type: optional<string>
+ docs: Opaque cursor returned by the previous page.
+ response: EvaluatorsPage
+ errors:
+ - errors.BadRequestError
+ - errors.UnauthorizedError
+ - errors.AccessDeniedError
+ - errors.MethodNotAllowedError
+ - errors.TooManyRequestsError
+ - errors.InternalServerError
+
+ get:
+ docs: |
+ Get one evaluator by its stable identifier.
+
+ The response includes the evaluator's latest definition and version metadata flattened into the evaluator object, plus associated evaluation rules. Use the version-history endpoint when older definitions are needed.
+ method: GET
+ path: /{evaluatorId}
+ path-parameters:
+ evaluatorId:
+ type: string
+ docs: Stable evaluator identifier returned by the evaluator endpoints.
+ response: Evaluator
+ errors:
+ - errors.UnauthorizedError
+ - errors.AccessDeniedError
+ - errors.NotFoundError
+ - errors.MethodNotAllowedError
+ - errors.TooManyRequestsError
+ - errors.InternalServerError
+
+ update:
+ docs: |
+ Update an evaluator by its stable identifier.
+
+ Provide only the top-level fields to change. Metadata-only changes do not create a version. Evaluator type cannot change.
+
+ Definition fields are flattened into the request. To replace a definition, include `type` and every definition field for that type. Definition fields are replaced as a complete unit rather than merged. For LLM-as-a-judge evaluators, omitting or setting `modelConfig` to `null` selects the project's default evaluation model.
+
+ Replacing a definition automatically returns an evaluator paused by an invalid or missing model configuration to `active`, while preserving its id and evaluation-rule assignments. Pauses caused by provider authentication, billing, connectivity, account state, or an unknown legacy reason remain paused until the explicit reactivation check succeeds.
+ method: PATCH
+ path: /{evaluatorId}
+ path-parameters:
+ evaluatorId:
+ type: string
+ docs: Stable evaluator identifier returned by the evaluator endpoints.
+ request: UpdateEvaluatorRequest
+ response: Evaluator
+ errors:
+ - errors.BadRequestError
+ - errors.UnauthorizedError
+ - errors.AccessDeniedError
+ - errors.NotFoundError
+ - errors.MethodNotAllowedError
+ - errors.ConflictError
+ - errors.PreconditionFailedError
+ - errors.TooManyRequestsError
+ - errors.InternalServerError
+
+ delete:
+ docs: |
+ Delete an evaluator and all of its stored versions.
+
+ Associated evaluation-rule assignments are also removed. Scores already produced by the evaluator are preserved.
+ method: DELETE
+ path: /{evaluatorId}
+ path-parameters:
+ evaluatorId:
+ type: string
+ docs: Stable evaluator identifier returned by the evaluator endpoints.
+ response: DeletedEvaluator
+ errors:
+ - errors.UnauthorizedError
+ - errors.AccessDeniedError
+ - errors.NotFoundError
+ - errors.MethodNotAllowedError
+ - errors.TooManyRequestsError
+ - errors.InternalServerError
+
+ listVersions:
+ docs: |
+ List an evaluator's version history in newest-first order.
+
+ This endpoint is intended for history and audit use cases. Ordinary clients can use the flattened `version` and definition fields on the evaluator response.
+ method: GET
+ path: /{evaluatorId}/versions
+ path-parameters:
+ evaluatorId:
+ type: string
+ docs: Stable evaluator identifier returned by the evaluator endpoints.
+ request:
+ name: ListEvaluatorVersionsRequest
+ query-parameters:
+ limit:
+ type: optional<integer>
+ docs: Maximum number of versions to return. Defaults to `50` and cannot exceed `100`.
+ cursor:
+ type: optional<string>
+ docs: Opaque cursor returned by the previous page.
+ response: EvaluatorVersionsPage
+ errors:
+ - errors.BadRequestError
+ - errors.UnauthorizedError
+ - errors.AccessDeniedError
+ - errors.NotFoundError
+ - errors.MethodNotAllowedError
+ - errors.TooManyRequestsError
+ - errors.InternalServerError
+
+types:
+ Creator:
+ docs: User who created the resource, or `null` when no user can be resolved.
+ properties:
+ id:
+ type: string
+ docs: User identifier.
+ name:
+ type: nullable<string>
+ docs: User name, or `null` when unavailable.
+
+ EvaluatorModelConfig:
+ docs: |
+ Explicit model configuration for an evaluator.
+
+ If the complete `modelConfig` is `null`, Langfuse uses the project's default evaluation model.
+ If provided, the model must be available to the project when the evaluator or evaluation rule is enabled.
+ properties:
+ provider:
+ type: string
+ docs: |
+ Provider identifier, for example `openai` or `anthropic`.
+
+ To discover valid values for the current project, call `GET /api/public/llm-connections` and use one of the returned `provider` values.
+ model:
+ type: string
+ docs: Model identifier exposed by the provider, for example `gpt-4.1-mini`.
+ examples:
+ - value:
+ provider: openai
+ model: gpt-4.1-mini
+
+ EvaluatorChatMessage:
+ docs: One user chat message in an evaluator prompt.
+ properties:
+ role:
+ type: literal<"user">
+ content:
+ type: string
+ docs: Message content. Evaluator variables use `{{variable}}` syntax.
+
+ EvaluatorChatPrompt:
+ docs: A list containing exactly one user chat message.
+ type: list<EvaluatorChatMessage>
+ validation:
+ minItems: 1
+ maxItems: 1
+
+ EvaluatorChatPromptInput:
+ docs: A user prompt string, or a list containing exactly one user chat message.
+ discriminated: false
+ union:
+ - string
+ - EvaluatorChatPrompt
+
+ EvaluatorVersionBase:
+ properties:
+ id:
+ type: string
+ docs: Stable identifier of this evaluator version.
+ version:
+ type: integer
+ docs: Monotonically increasing evaluator version number.
+ createdAt:
+ type: datetime
+ docs: Timestamp when this evaluator version was created.
+ createdBy:
+ type: nullable<Creator>
+ docs: User who created this version, or `null` when no user can be resolved.
+
+ LlmAsJudgeEvaluatorVersion:
+ extends: EvaluatorVersionBase
+ properties:
+ type:
+ type: literal<"llm_as_judge">
+ docs: Evaluator type.
+ prompt:
+ type: EvaluatorChatPrompt
+ docs: The single user chat message used during evaluation.
+ variables:
+ type: list<string>
+ docs: |
+ Variables extracted from the prompt and available for evaluation-rule mappings.
+
+ Every variable must be mapped exactly once when a rule provides an explicit mapping.
+ variableMapping:
+ type: nullable<list<evals.PromptVariableMappingRead>>
+ docs: Default variable mapping for this evaluator version, or `null` when no default is configured.
+ modelConfig:
+ type: nullable<EvaluatorModelConfig>
+ docs: Explicit model configuration, or `null` when the project's default evaluation model is used.
+ outputDefinition:
+ type: evals.PublicEvaluatorOutputDefinition
+ docs: |
+ Structured output schema returned by this evaluator.
+
+ Responses include `dataType` and omit the internal output-definition persistence version.
+
+ CodeEvaluatorVersion:
+ extends: EvaluatorVersionBase
+ properties:
+ type:
+ type: literal<"code">
+ docs: Evaluator type.
+ sourceCode:
+ type: string
+ docs: Source code executed for each matched observation.
+ sourceCodeLanguage:
+ type: evals.CodeEvaluatorSourceCodeLanguage
+ docs: Runtime language used to execute the source code.
+
+ EvaluatorVersion:
+ discriminant: type
+ union:
+ llm_as_judge:
+ type: LlmAsJudgeEvaluatorVersion
+ code:
+ type: CodeEvaluatorVersion
+
+ EvaluationRuleAssignment:
+ docs: Evaluation-rule assignment associated with this evaluator.
+ properties:
+ evaluationRuleId:
+ type: string
+ docs: Stable identifier of the assigned evaluation rule.
+ variableMappingOverride:
+ type: optional<list<evals.PromptVariableMapping>>
+ docs: Rule-specific variable mapping override. Omitted when the evaluator's latest default mapping is inherited. Legacy mappings use the explicit `legacy` variant.
+
+ EvaluatorBase:
+ properties:
+ id:
+ type: string
+ docs: Stable identifier of this evaluator across all versions.
+ name:
+ type: string
+ docs: Human-readable evaluator name. Names are not identifiers and do not need to be unique.
+ description:
+ type: nullable<string>
+ docs: Optional human-readable evaluator description.
+ createdBy:
+ type: nullable<Creator>
+ docs: User who created this evaluator, or `null` when no user can be resolved.
+ status:
+ type: EvaluatorStatus
+ docs: Effective evaluator status after Langfuse validates its runtime configuration.
+ pausedAt:
+ type: nullable<datetime>
+ docs: Timestamp when the evaluator was paused, otherwise `null`.
+ pausedReason:
+ type: nullable<string>
+ docs: Machine-readable reason when `status=paused`, otherwise `null`.
+ pausedMessage:
+ type: nullable<string>
+ docs: Human-readable explanation when `status=paused`, otherwise `null`.
+ evaluationRuleAssignments:
+ type: list<EvaluationRuleAssignment>
+ docs: All modern and legacy evaluation-rule assignments in newest-assignment-first order. Rule-specific mappings are exposed as `variableMappingOverride`; inherited defaults are omitted.
+ createdAt:
+ type: datetime
+ docs: Timestamp when the evaluator was created.
+ updatedAt:
+ type: datetime
+ docs: Timestamp when the evaluator was last updated.
+ versionId:
+ type: string
+ docs: Stable identifier of the latest evaluator version.
+ version:
+ type: integer
+ docs: Monotonically increasing latest evaluator version number.
+ versionCreatedAt:
+ type: datetime
+ docs: Timestamp when the latest evaluator version was created.
+ versionCreatedBy:
+ type: nullable<Creator>
+ docs: User who created the latest evaluator version, or `null` when no user can be resolved.
+
+ LlmAsJudgeEvaluator:
+ extends: EvaluatorBase
+ properties:
+ type:
+ type: literal<"llm_as_judge">
+ docs: Evaluator type.
+ prompt:
+ type: EvaluatorChatPrompt
+ docs: The single user chat message used by the latest evaluator version.
+ variables:
+ type: list<string>
+ docs: Variables extracted from the latest prompt and available for evaluation-rule mappings.
+ variableMapping:
+ type: nullable<list<evals.PromptVariableMappingRead>>
+ docs: Default variable mapping for the latest version, or `null` when no default is configured.
+ modelConfig:
+ type: nullable<EvaluatorModelConfig>
+ docs: Explicit model configuration for the latest version, or `null` when the project's default evaluation model is used.
+ outputDefinition:
+ type: evals.PublicEvaluatorOutputDefinition
+ docs: Structured output schema returned by the latest evaluator version.
+
+ CodeEvaluator:
+ extends: EvaluatorBase
+ properties:
+ type:
+ type: literal<"code">
+ docs: Evaluator type.
+ sourceCode:
+ type: string
+ docs: Source code executed by the latest evaluator version.
+ sourceCodeLanguage:
+ type: evals.CodeEvaluatorSourceCodeLanguage
+ docs: Runtime language used to execute the latest source code.
+
+ Evaluator:
+ docs: |
+ One evaluator that can be used for scoring.
+
+ An evaluator describes **how** to score data. Associated evaluation rules describe **which** live objects should be evaluated.
+ discriminant: type
+ union:
+ llm_as_judge:
+ type: LlmAsJudgeEvaluator
+ code:
+ type: CodeEvaluator
+
+ EvaluatorStatus:
+ docs: |
+ Effective evaluator runtime status.
+
+ - `active`: the evaluator can run.
+ - `paused`: Langfuse paused execution until the underlying issue is resolved.
+ enum:
+ - active
+ - paused
+
+ CreateLlmAsJudgeEvaluatorRequest:
+ properties:
+ name:
+ type: string
+ docs: Human-readable evaluator name. Names are not identifiers and do not need to be unique.
+ description:
+ type: optional<nullable<string>>
+ docs: Optional human-readable evaluator description.
+ type:
+ type: literal<"llm_as_judge">
+ docs: Evaluator type.
+ prompt:
+ type: EvaluatorChatPromptInput
+ docs: User prompt string shortcut or a list containing exactly one user chat message.
+ modelConfig:
+ type: optional<nullable<EvaluatorModelConfig>>
+ docs: Explicit model configuration. Set to `null` or omit to use the project's default evaluation model.
+ variableMapping:
+ type: optional<nullable<list<evals.PromptVariableMappingInput>>>
+ docs: Default prompt-variable mapping, or `null` when no default is configured.
+ outputDefinition:
+ type: evals.EvaluatorOutputDefinition
+ docs: Structured output schema returned by this evaluator. Do not include an internal persistence version.
+
+ CreateCodeEvaluatorRequest:
+ properties:
+ name:
+ type: string
+ docs: Human-readable evaluator name. Names are not identifiers and do not need to be unique.
+ description:
+ type: optional<nullable<string>>
+ docs: Optional human-readable evaluator description.
+ type:
+ type: literal<"code">
+ docs: Evaluator type.
+ sourceCode:
+ type: string
+ docs: Source code executed for each matched observation.
+ sourceCodeLanguage:
+ type: evals.CodeEvaluatorSourceCodeLanguage
+ docs: Runtime language used to execute the source code.
+
+ CreateEvaluatorRequest:
+ docs: Initial evaluator definition with metadata and definition fields at the same level. The returned evaluator starts at version `1`.
+ discriminant: type
+ union:
+ llm_as_judge:
+ type: CreateLlmAsJudgeEvaluatorRequest
+ code:
+ type: CreateCodeEvaluatorRequest
+
+ UpdateEvaluatorMetadataRequest:
+ docs: Metadata-only update. At least one field must be provided.
+ properties:
+ name:
+ type: optional<string>
+ docs: New human-readable evaluator name.
+ description:
+ type: optional<nullable<string>>
+ docs: New description. Set to `null` to clear it.
+
+ UpdateLlmAsJudgeEvaluatorRequest:
+ properties:
+ name:
+ type: optional<string>
+ docs: New human-readable evaluator name.
+ description:
+ type: optional<nullable<string>>
+ docs: New description. Set to `null` to clear it.
+ type:
+ type: literal<"llm_as_judge">
+ docs: Evaluator type. The type of an existing evaluator cannot change.
+ prompt:
+ type: EvaluatorChatPromptInput
+ docs: Complete replacement user prompt string or a list containing exactly one user chat message.
+ modelConfig:
+ type: optional<nullable<EvaluatorModelConfig>>
+ docs: Explicit model configuration. Set to `null` or omit to use the project's default evaluation model.
+ variableMapping:
+ type: optional<nullable<list<evals.PromptVariableMappingInput>>>
+ docs: Complete replacement default variable mapping, or `null` when no default is configured.
+ outputDefinition:
+ type: evals.EvaluatorOutputDefinition
+ docs: Complete replacement output schema. Do not include an internal persistence version.
+
+ UpdateCodeEvaluatorRequest:
+ properties:
+ name:
+ type: optional<string>
+ docs: New human-readable evaluator name.
+ description:
+ type: optional<nullable<string>>
+ docs: New description. Set to `null` to clear it.
+ type:
+ type: literal<"code">
+ docs: Evaluator type. The type of an existing evaluator cannot change.
+ sourceCode:
+ type: string
+ docs: Complete replacement source code.
+ sourceCodeLanguage:
+ type: evals.CodeEvaluatorSourceCodeLanguage
+ docs: Runtime language used to execute the source code.
+
+ UpdateEvaluatorRequest:
+ docs: Metadata-only update or complete flattened definition replacement. At least one field must be provided.
+ discriminated: false
+ union:
+ - UpdateEvaluatorMetadataRequest
+ - UpdateLlmAsJudgeEvaluatorRequest
+ - UpdateCodeEvaluatorRequest
+
+ EvaluatorsPage:
+ properties:
+ data:
+ type: list<Evaluator>
+ docs: Evaluators for this page.
+ meta:
+ type: CursorMeta
+ docs: Cursor pagination metadata.
+
+ EvaluatorVersionsPage:
+ properties:
+ data:
+ type: list<EvaluatorVersion>
+ docs: Evaluator versions for this page in newest-first order.
+ meta:
+ type: CursorMeta
+ docs: Cursor pagination metadata.
+
+ CursorMeta:
+ properties:
+ cursor:
+ type: optional<string>
+ docs: Opaque cursor for the next page. Omitted when there is no next page.
+
+ DeletedEvaluator:
+ docs: Confirmation returned after successful evaluator deletion.
+ properties:
+ id:
+ type: string
+ docs: Identifier of the deleted evaluator.
diff --git a/fern/apis/server/definition/unstable/commons.yml b/fern/apis/server/definition/unstable/commons.yml
index b616d1435f..48e0c0a72a 100644
--- a/fern/apis/server/definition/unstable/commons.yml
+++ b/fern/apis/server/definition/unstable/commons.yml
@@ -2,7 +2,7 @@
types:
EvaluatorType:
docs: |
- The evaluator engine type.
+ The evaluator type.
The unstable public API supports LLM-as-a-judge and code evaluators.
enum:
@@ -41,7 +41,7 @@ types:
- inactive
- paused
- EvaluationRuleMappingSource:
+ PromptVariableMappingSource:
docs: |
Source field used to populate a prompt variable.
@@ -422,9 +422,9 @@ types:
type: optional<string>
docs: Ignored placeholder value. Clients may omit it or send an empty string.
- EvaluationRuleMapping:
+ PromptVariableMappingInput:
docs: |
- Maps one evaluator variable to one source field from the target object.
+ Connects one prompt variable to the available evaluation data.
Manual mappings are used for `llm_as_judge` evaluators. `code` evaluators use a fixed runtime mapping managed by Langfuse.
@@ -439,8 +439,8 @@ types:
Recovery guidance:
- `invalid_variable_mapping`: the variable name is unknown for this evaluator, or the selected `source` is not valid for the chosen `target`
- - `missing_variable_mapping`: one or more LLM-as-judge evaluator variables are not mapped yet
- - `duplicate_variable_mapping`: the same evaluator variable appears more than once
+ - `missing_variable_mapping`: one or more LLM-as-judge prompt variables are not mapped yet
+ - `duplicate_variable_mapping`: the same prompt variable appears more than once
- `invalid_json_path`: the JSONPath expression is malformed. Remove it or correct it.
properties:
variable:
@@ -450,7 +450,7 @@ types:
Example: for the prompt `Judge {{input}} against {{output}}`, use `input` and `output`.
source:
- type: EvaluationRuleMappingSource
+ type: PromptVariableMappingSource
docs: |
Source field that should populate the prompt variable.
@@ -481,23 +481,17 @@ types:
variable: expected_output
source: expected_output
- EvaluationRuleReadMapping:
+ PromptVariableMappingRead:
docs: |
- Maps an evaluator variable to target data, as returned by read endpoints.
+ Connects one prompt variable to data from a target.
- Reads are more permissive than writes in two ways, so that data configured before a
- migration stays readable and repairable:
- - `source` is `null` when a mapping was never fully configured. Set a concrete `source` to fix it.
- - every `source` value is allowed regardless of the rule `target`, because an evaluator's
- default mapping is stored independently of the rules that use it.
-
- Requests always require a concrete `source` that is valid for the selected `target`.
+ `source` is `null` when the mapping is incomplete.
properties:
variable:
type: string
docs: Prompt variable name without braces.
source:
- type: nullable<EvaluationRuleMappingSource>
+ type: nullable<PromptVariableMappingSource>
docs: Source field populating the variable, or `null` when the mapping is incomplete.
jsonPath:
type: optional<string>
diff --git a/fern/apis/server/definition/unstable/evaluation-rules.yml b/fern/apis/server/definition/unstable/evaluation-rules.yml
index 5de05f3a9a..ccf0d16187 100644
--- a/fern/apis/server/definition/unstable/evaluation-rules.yml
+++ b/fern/apis/server/definition/unstable/evaluation-rules.yml
@@ -9,6 +9,9 @@ service:
base-path: /api/public/unstable
endpoints:
create:
+ availability:
+ status: deprecated
+ message: "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4."
docs: |
Create an evaluation rule.
@@ -236,6 +239,9 @@ service:
updatedAt: "2026-03-30T09:30:00.000Z"
list:
+ availability:
+ status: deprecated
+ message: "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4."
docs: |
List evaluation rules in the authenticated project.
@@ -261,6 +267,9 @@ service:
- errors.InternalServerError
get:
+ availability:
+ status: deprecated
+ message: "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4."
docs: |
Get one evaluation rule by its identifier.
@@ -282,6 +291,9 @@ service:
- errors.InternalServerError
update:
+ availability:
+ status: deprecated
+ message: "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4."
docs: |
Update an evaluation rule.
@@ -323,6 +335,9 @@ service:
- errors.InternalServerError
delete:
+ availability:
+ status: deprecated
+ message: "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4."
docs: |
Delete an evaluation rule.
@@ -352,7 +367,7 @@ types:
- which evaluator should be used
- which target objects should trigger scoring
- how often scoring should run
- - which target fields should populate each evaluator variable
+ - which target fields should populate each prompt variable
- whether the deployment is active, inactive, or paused
Important status semantics:
@@ -414,7 +429,7 @@ types:
type: list<commons.EvaluationRuleFilter>
docs: List of filter conditions used to decide whether a target should be evaluated.
mapping:
- type: list<commons.EvaluationRuleReadMapping>
+ type: list<commons.PromptVariableMappingRead>
docs: Deprecated compatibility alias containing the effective mapping for `evaluators[0]`.
examples:
- value:
@@ -456,9 +471,9 @@ types:
LegacyEvaluationRule:
docs: |
- Legacy trace- or dataset-level evaluation rule returned by list and get for migration.
+ **Deprecated:** Legacy trace- or dataset-level evaluation rule returned by list and get for migration.
- This resource is read-only through the unstable public API. Its mapping preserves the trace, dataset item, or named observation selected for each evaluator variable.
+ This resource is read-only through the unstable public API. Its mapping preserves the trace, dataset item, or named observation selected for each prompt variable.
extends: EvaluationRuleBase
properties:
evaluators:
@@ -476,7 +491,7 @@ types:
type: list<commons.EvaluationRuleFilter>
docs: Stored filters used by the legacy trace or dataset rule.
mapping:
- type: list<LegacyEvaluationRuleMapping>
+ type: list<LegacyPromptVariableMapping>
docs: Stored variable mappings, including the trace, dataset item, or named observation selected for each variable.
ReadableEvaluationRule:
@@ -504,7 +519,7 @@ types:
- reference an existing evaluator family by `evaluator.name` and `evaluator.type`
- choose `target=observation` or `target=experiment`
- if `target=experiment` and you want a dataset filter, call `GET /api/public/v2/datasets` first and use dataset `id` values in `filter[].value`
- - for `llm_as_judge`, fetch or inspect the evaluator first and provide a complete variable mapping for every evaluator variable
+ - for `llm_as_judge`, fetch or inspect the evaluator first and provide a complete mapping for every prompt variable
- for `code`, do not send variables or mappings; Langfuse stores the fixed code runtime mapping automatically
- optionally narrow execution with `filter`
- set `enabled=true` only when you want live execution immediately
@@ -564,12 +579,12 @@ types:
Each filter object must use a column that is valid for that `target`.
For `target=experiment`, `column=datasetId` expects dataset `id` values from `GET /api/public/v2/datasets`, not dataset names.
mapping:
- type: list<commons.EvaluationRuleMapping>
+ type: list<commons.PromptVariableMappingInput>
docs: |
LLM-as-judge variable mappings.
- Every evaluator variable must appear exactly once.
- Build this list from the evaluator `variables` array returned by the evaluator endpoints.
+ Every prompt variable must appear exactly once.
+ Build this list from the evaluator's `variables` array.
CreateCodeEvaluationRuleRequest:
properties:
@@ -612,7 +627,7 @@ types:
- If you only want to rename the rule or change sampling, send just those fields.
- To add, remove, or remap evaluators, send `evaluators`. It replaces the whole assignment set, so include every evaluator the rule should keep.
- `evaluators` cannot be combined with the deprecated `evaluator`/`mapping` pair, which only ever addressed the first assignment.
- - If you change to an LLM-as-judge `evaluator`, send a fresh `mapping` unless you are certain the existing mapping still matches the evaluator variables.
+ - If you change to an LLM-as-judge `evaluator`, send a fresh `mapping` unless you are certain the existing mapping still matches the prompt variables.
- If you change `target` for an LLM-as-judge rule, usually send both `filter` and `mapping` in the same request.
- For code evaluator rules, omit `mapping`; Langfuse stores the fixed code runtime mapping automatically.
- If you change an experiment `datasetId` filter, call `GET /api/public/v2/datasets` and use dataset `id` values from that response.
@@ -650,7 +665,7 @@ types:
For `target=experiment`, `column=datasetId` expects dataset `id` values from `GET /api/public/v2/datasets`, not dataset names.
mapping:
- type: optional<list<commons.EvaluationRuleMapping>>
+ type: optional<list<commons.PromptVariableMappingInput>>
docs: |
Updated LLM-as-judge variable mappings.
@@ -682,7 +697,7 @@ types:
docs: Evaluator family name.
type:
type: optional<LlmAsJudgeEvaluatorType>
- docs: Evaluator engine type. Defaults to `llm_as_judge` when omitted.
+ docs: Evaluator type. Defaults to `llm_as_judge` when omitted.
CodeEvaluationRuleEvaluatorReference:
docs: Code evaluator family reference used when creating an evaluation rule.
@@ -713,22 +728,23 @@ types:
docs: Evaluator family name.
type:
type: commons.EvaluatorType
- docs: Evaluator engine type.
+ docs: Evaluator type.
EvaluationRuleEvaluatorAssignment:
properties:
evaluator:
type: EvaluationRuleEvaluator
mapping:
- type: nullable<list<commons.EvaluationRuleReadMapping>>
+ type: nullable<list<commons.PromptVariableMappingRead>>
docs: Rule-specific override, or `null` to inherit the evaluator version's default mapping.
LegacyEvaluationRuleEvaluatorAssignment:
+ docs: "**Deprecated:** Evaluator assignment on a legacy trace or dataset rule."
properties:
evaluator:
type: EvaluationRuleEvaluator
mapping:
- type: nullable<list<LegacyEvaluationRuleMapping>>
+ type: nullable<list<LegacyPromptVariableMapping>>
EvaluationRuleTimeScope:
enum:
@@ -736,12 +752,13 @@ types:
- EXISTING
LegacyEvaluationRuleTarget:
+ docs: "**Deprecated:** Legacy trace or dataset evaluation target."
enum:
- trace
- dataset
- LegacyEvaluationRuleMapping:
- docs: Maps one evaluator variable to a trace, dataset item, or field on a named observation in a legacy rule.
+ LegacyPromptVariableMapping:
+ docs: "**Deprecated:** Connects one prompt variable to data from a trace, dataset item, or named observation in a legacy rule."
properties:
variable:
type: string
@@ -756,6 +773,7 @@ types:
type: optional<string>
LegacyEvaluationObject:
+ docs: "**Deprecated:** Legacy Langfuse object used by a trace or dataset evaluation rule."
enum:
- trace
- span
@@ -775,7 +793,7 @@ types:
evaluator:
type: CreateEvaluationRuleEvaluatorReference
mapping:
- type: optional<list<commons.EvaluationRuleMapping>>
+ type: optional<list<commons.PromptVariableMappingInput>>
docs: Optional rule-specific override. Omit to inherit the evaluator version's default mapping. Omit for code evaluators.
CreateEvaluationRuleEvaluatorReference:
diff --git a/fern/apis/server/definition/unstable/evaluators.yml b/fern/apis/server/definition/unstable/evaluators.yml
index e5a6eee91a..622adf79c1 100644
--- a/fern/apis/server/definition/unstable/evaluators.yml
+++ b/fern/apis/server/definition/unstable/evaluators.yml
@@ -9,6 +9,9 @@ service:
base-path: /api/public/unstable
endpoints:
create:
+ availability:
+ status: deprecated
+ message: "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluators` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4."
docs: |
Create an evaluator in the authenticated project.
@@ -257,6 +260,9 @@ service:
updatedAt: "2026-03-30T09:15:00.000Z"
list:
+ availability:
+ status: deprecated
+ message: "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluators` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4."
docs: |
List the evaluators available to the authenticated project.
@@ -284,6 +290,9 @@ service:
- errors.InternalServerError
get:
+ availability:
+ status: deprecated
+ message: "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluators` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4."
docs: |
Get one evaluator by `id`.
@@ -305,6 +314,9 @@ service:
- errors.InternalServerError
delete:
+ availability:
+ status: deprecated
+ message: "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluators` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4."
docs: |
Delete an evaluator.
@@ -378,7 +390,7 @@ types:
LLM evaluators require every variable to be mapped exactly once. Code evaluators always expose the fixed runtime payload fields and Langfuse maps them automatically.
mapping:
- type: nullable<list<commons.EvaluationRuleReadMapping>>
+ type: nullable<list<commons.PromptVariableMappingRead>>
docs: |
Default variable mapping for this evaluator version, or `null` when no default is configured.
@@ -462,7 +474,7 @@ types:
type: optional<nullable<commons.EvaluatorModelConfig>>
docs: Optional explicit model configuration. Omit or set to `null` to use the project default evaluation model.
mapping:
- type: optional<list<commons.EvaluationRuleMapping>>
+ type: optional<list<commons.PromptVariableMappingInput>>
docs: Optional default variable mapping inherited by rule assignments that do not provide an override.
CreateCodeEvaluatorRequest:
diff --git a/packages/shared/src/features/evals/codeEvalConstants.ts b/packages/shared/src/features/evals/codeEvalConstants.ts
new file mode 100644
index 0000000000..7b920c1025
--- /dev/null
+++ b/packages/shared/src/features/evals/codeEvalConstants.ts
@@ -0,0 +1 @@
+export const CODE_EVAL_SOURCE_MAX_BYTES = 256 * 1024;
diff --git a/packages/shared/src/features/evals/evalConfigBlocking.ts b/packages/shared/src/features/evals/evalConfigBlocking.ts
index b6db3165d0..bb599ff227 100644
--- a/packages/shared/src/features/evals/evalConfigBlocking.ts
+++ b/packages/shared/src/features/evals/evalConfigBlocking.ts
@@ -112,6 +112,22 @@ export function getEvaluatorBlockMetadata(
return EVALUATOR_BLOCK_METADATA[reason];
}
+const DEFINITION_UPDATE_RECOVERABLE_BLOCK_REASONS: ReadonlySet<EvaluatorBlockReason> =
+ new Set([
+ EvaluatorBlockReason.LLM_CONNECTION_MISSING,
+ EvaluatorBlockReason.DEFAULT_EVAL_MODEL_MISSING,
+ EvaluatorBlockReason.EVAL_MODEL_CONFIG_INVALID,
+ EvaluatorBlockReason.EVAL_MODEL_UNAVAILABLE,
+ ]);
+
+export function isEvaluatorBlockReasonRecoverableByDefinitionUpdate(
+ reason: EvaluatorBlockReason | null | undefined,
+) {
+ return reason
+ ? DEFINITION_UPDATE_RECOVERABLE_BLOCK_REASONS.has(reason)
+ : false;
+}
+
export function getEvaluatorBlockResolutionPath(params: {
projectId: string;
blockReason: EvaluatorBlockReason;
diff --git a/packages/shared/src/features/evals/outputDefinition.test.ts b/packages/shared/src/features/evals/outputDefinition.test.ts
new file mode 100644
index 0000000000..f17701309a
--- /dev/null
+++ b/packages/shared/src/features/evals/outputDefinition.test.ts
@@ -0,0 +1,63 @@
+import { describe, expect, it } from "vitest";
+import {
+ EvalOutputDefinitionSchema,
+ PersistedEvalOutputDefinitionSchema,
+} from "./outputDefinition";
+
+describe("evaluator output definitions", () => {
+ it("uses the same versionless shape in the service and persistence", () => {
+ const definition = EvalOutputDefinitionSchema.parse({
+ dataType: "BOOLEAN",
+ reasoning: { description: "Why the verdict was selected" },
+ score: { description: "Whether the input passes" },
+ });
+
+ expect(PersistedEvalOutputDefinitionSchema.parse(definition)).toEqual(
+ definition,
+ );
+ expect(definition).not.toHaveProperty("version");
+ });
+
+ it("defaults omitted score descriptions", () => {
+ expect(
+ EvalOutputDefinitionSchema.parse({
+ dataType: "NUMERIC",
+ reasoning: {},
+ score: { minValue: 0, maxValue: 1 },
+ }),
+ ).toEqual({
+ dataType: "NUMERIC",
+ reasoning: { description: "" },
+ score: { description: "", minValue: 0, maxValue: 1 },
+ });
+ });
+
+ it("continues to read existing definitions that contain a version", () => {
+ expect(
+ PersistedEvalOutputDefinitionSchema.parse({
+ version: 2,
+ dataType: "BOOLEAN",
+ reasoning: { description: "Why the verdict was selected" },
+ score: { description: "Whether the input passes" },
+ }),
+ ).toEqual({
+ dataType: "BOOLEAN",
+ reasoning: { description: "Why the verdict was selected" },
+ score: { description: "Whether the input passes" },
+ });
+ });
+
+ it("applies categorical validation before persistence", () => {
+ expect(
+ EvalOutputDefinitionSchema.safeParse({
+ dataType: "CATEGORICAL",
+ reasoning: { description: "Why the category was selected" },
+ score: {
+ description: "Selected category",
+ categories: ["duplicate", "duplicate"],
+ shouldAllowMultipleMatches: false,
+ },
+ }).success,
+ ).toBe(false);
+ });
+});
diff --git a/packages/shared/src/features/evals/outputDefinition.ts b/packages/shared/src/features/evals/outputDefinition.ts
index ae64fcb547..6d03c5d5d3 100644
--- a/packages/shared/src/features/evals/outputDefinition.ts
+++ b/packages/shared/src/features/evals/outputDefinition.ts
@@ -19,7 +19,7 @@ export type LegacyEvalOutputDefinition = z.infer<
>;
const EvalOutputFieldDefinitionSchema = z.object({
- description: z.string().trim().min(1),
+ description: z.string().trim().default(""),
});
export const MinimumCategoricalCategoryCount = 2;
@@ -81,68 +81,66 @@ const NumericEvalOutputScoreDefinitionSchema =
},
);
-export const NumericEvalOutputDefinitionV2Schema = z.object({
- version: z.literal(2),
+const NumericEvalOutputDefinitionSchema = z.object({
dataType: z.literal(ScoreDataTypeEnum.NUMERIC),
reasoning: EvalOutputFieldDefinitionSchema,
score: NumericEvalOutputScoreDefinitionSchema,
});
-export type NumericEvalOutputDefinitionV2 = z.infer<
- typeof NumericEvalOutputDefinitionV2Schema
->;
-export const BooleanEvalOutputDefinitionV2Schema = z.object({
- version: z.literal(2),
+const BooleanEvalOutputDefinitionSchema = z.object({
dataType: z.literal(ScoreDataTypeEnum.BOOLEAN),
reasoning: EvalOutputFieldDefinitionSchema,
score: EvalOutputFieldDefinitionSchema,
});
-export type BooleanEvalOutputDefinitionV2 = z.infer<
- typeof BooleanEvalOutputDefinitionV2Schema
->;
-export const CategoricalEvalOutputDefinitionV2Schema = z
+function validateCategoricalOutputDefinition(
+ value: { score: { categories: string[] } },
+ ctx: z.RefinementCtx,
+) {
+ getCategoricalCategoryRuleViolations(value.score.categories).forEach(
+ (violation) => {
+ switch (violation.type) {
+ case "minimum_count":
+ ctx.addIssue({
+ code: "custom",
+ message: getMinimumCategoricalCategoriesMessage(),
+ path: ["score", "categories"],
+ });
+ return;
+ case "duplicate_value":
+ ctx.addIssue({
+ code: "custom",
+ message: "Categories must be unique",
+ path: ["score", "categories", violation.index],
+ });
+ return;
+ }
+ },
+ );
+}
+
+const CategoricalEvalOutputDefinitionSchema = z
.object({
- version: z.literal(2),
dataType: z.literal(ScoreDataTypeEnum.CATEGORICAL),
reasoning: EvalOutputFieldDefinitionSchema,
score: z.object({
- description: z.string().trim().min(1),
+ description: z.string().trim().default(""),
categories: z.array(EvalCategoricalCategorySchema),
shouldAllowMultipleMatches: z.boolean().default(false),
}),
})
- .superRefine((value, ctx) => {
- getCategoricalCategoryRuleViolations(value.score.categories).forEach(
- (violation) => {
- switch (violation.type) {
- case "minimum_count":
- ctx.addIssue({
- code: "custom",
- message: getMinimumCategoricalCategoriesMessage(),
- path: ["score", "categories"],
- });
- return;
- case "duplicate_value":
- ctx.addIssue({
- code: "custom",
- message: "Categories must be unique",
- path: ["score", "categories", violation.index],
- });
- return;
- }
- },
- );
- });
-export type CategoricalEvalOutputDefinitionV2 = z.infer<
- typeof CategoricalEvalOutputDefinitionV2Schema
->;
+ .superRefine(validateCategoricalOutputDefinition);
+
+export const EvalOutputDefinitionSchema = z.union([
+ NumericEvalOutputDefinitionSchema,
+ BooleanEvalOutputDefinitionSchema,
+ CategoricalEvalOutputDefinitionSchema,
+]);
+export type EvalOutputDefinition = z.infer<typeof EvalOutputDefinitionSchema>;
export const PersistedEvalOutputDefinitionSchema = z.union([
LegacyEvalOutputDefinitionSchema,
- NumericEvalOutputDefinitionV2Schema,
- BooleanEvalOutputDefinitionV2Schema,
- CategoricalEvalOutputDefinitionV2Schema,
+ EvalOutputDefinitionSchema,
]);
export type PersistedEvalOutputDefinition = z.infer<
typeof PersistedEvalOutputDefinitionSchema
@@ -196,7 +194,7 @@ export type EvalOutputResult =
export function resolvePersistedEvalOutputDefinition(
outputDefinition: PersistedEvalOutputDefinition,
): ResolvedEvalOutputDefinition {
- if (!("version" in outputDefinition)) {
+ if (!("dataType" in outputDefinition)) {
return {
dataType: ScoreDataTypeEnum.NUMERIC,
reasoningDescription: outputDefinition.reasoning,
@@ -238,8 +236,7 @@ export function createNumericEvalOutputDefinition(params: {
minValue?: number;
maxValue?: number;
}) {
- return NumericEvalOutputDefinitionV2Schema.parse({
- version: 2,
+ return NumericEvalOutputDefinitionSchema.parse({
dataType: ScoreDataTypeEnum.NUMERIC,
reasoning: {
description: params.reasoningDescription,
@@ -256,8 +253,7 @@ export function createBooleanEvalOutputDefinition(params: {
reasoningDescription: string;
scoreDescription: string;
}) {
- return BooleanEvalOutputDefinitionV2Schema.parse({
- version: 2,
+ return BooleanEvalOutputDefinitionSchema.parse({
dataType: ScoreDataTypeEnum.BOOLEAN,
reasoning: {
description: params.reasoningDescription,
@@ -274,8 +270,7 @@ export function createCategoricalEvalOutputDefinition(params: {
categories: string[];
shouldAllowMultipleMatches?: boolean;
}) {
- return CategoricalEvalOutputDefinitionV2Schema.parse({
- version: 2,
+ return CategoricalEvalOutputDefinitionSchema.parse({
dataType: ScoreDataTypeEnum.CATEGORICAL,
reasoning: {
description: params.reasoningDescription,
diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts
index 79a61a1b02..db37c15a3d 100644
--- a/packages/shared/src/index.ts
+++ b/packages/shared/src/index.ts
@@ -39,6 +39,7 @@ export * from "./server/llm/promptToolConfig";
export * from "./features/evals/types";
export * from "./features/evals/evalExecutionMetadata";
export * from "./features/evals/outputDefinition";
+export * from "./features/evals/codeEvalConstants";
export {
extractValueFromObject,
extractValueFromObjectAsString,
diff --git a/packages/shared/src/server/evals/codeEvalDispatcherTypes.ts b/packages/shared/src/server/evals/codeEvalDispatcherTypes.ts
index bec59d4324..8581bd6e1e 100644
--- a/packages/shared/src/server/evals/codeEvalDispatcherTypes.ts
+++ b/packages/shared/src/server/evals/codeEvalDispatcherTypes.ts
@@ -4,8 +4,8 @@ import type {
CodeEvalTemplateVariable,
ToolCallForEval,
} from "../../features/evals/observationForEval";
-
-export const CODE_EVAL_SOURCE_MAX_BYTES = 256 * 1024;
+export { CODE_EVAL_SOURCE_MAX_BYTES } from "../../features/evals/codeEvalConstants";
+import { CODE_EVAL_SOURCE_MAX_BYTES } from "../../features/evals/codeEvalConstants";
export const CODE_EVAL_DISPATCH_PAYLOAD_MAX_BYTES = 5.5 * 1024 * 1024;
export const CODE_EVAL_DISPATCH_RESULT_MAX_BYTES = 256 * 1024;
export const CODE_EVAL_DOCS_URL =
diff --git a/web/public/generated/api/openapi.yml b/web/public/generated/api/openapi.yml
index e6435ad853..a2008c9afa 100644
--- a/web/public/generated/api/openapi.yml
+++ b/web/public/generated/api/openapi.yml
@@ -1597,99 +1597,41 @@ paths:
schema: {}
security: *ref_0
deprecated: true
- /api/public/experiments:
- get:
- description: |-
- List experiments with cursor-based pagination. Results are ordered by
- latest experiment activity descending.
- operationId: experiments_list
+ /api/public/v2/evaluation-rules:
+ post:
+ description: >-
+ Create an evaluation rule using stable evaluator identifiers.
+
+
+ An evaluation rule defines **which** incoming observations should be
+ evaluated and how prompt variables should be populated.
+
+
+ Key behavior:
+
+ - rule names are not identifiers and do not need to be unique
+
+ - rules always use the latest version of each associated evaluator
+
+ - an enabled rule requires at least one evaluator assignment
+
+ - omit `sampling` to evaluate every matching observation
+
+ - omit `filter` to match every incoming observation
+
+ - `datasetId` is the public filter name for selecting experiment
+ datasets
+ operationId: evaluationRules_create
tags:
- - Experiments
- parameters:
- - name: fields
- in: query
- description: |-
- Comma-separated list of field groups to include. Available groups:
- `core`, `metadata`, `scores`. If omitted, `core` is returned.
- required: false
- schema:
- type: string
- nullable: true
- - name: limit
- in: query
- description: Number of experiments to return per page. Maximum 100, default 50.
- required: false
- schema:
- type: integer
- nullable: true
- - name: scoreLimit
- in: query
- description: >-
- Number of scores to return per experiment when `fields=scores` is
- requested. Maximum 50, default 50.
- required: false
- schema:
- type: integer
- nullable: true
- - name: cursor
- in: query
- description: Versioned base64url cursor from the previous response page.
- required: false
- schema:
- type: string
- nullable: true
- - name: fromStartTime
- in: query
- description: Retrieve only experiments on or after this datetime.
- required: true
- schema:
- type: string
- format: date-time
- - name: toStartTime
- in: query
- description: Retrieve only experiments before this datetime.
- required: false
- schema:
- type: string
- format: date-time
- nullable: true
- - name: id
- in: query
- description: Comma-separated list of experiment IDs.
- required: false
- schema:
- type: string
- nullable: true
- - name: name
- in: query
- description: Comma-separated list of experiment names.
- required: false
- schema:
- type: string
- nullable: true
- - name: datasetId
- in: query
- description: Comma-separated list of dataset IDs.
- required: false
- schema:
- type: string
- nullable: true
- - name: filter
- in: query
- description: |-
- JSON string containing an array of structured filter conditions.
- Supported columns are `id`, `name`, and `datasetId`.
- required: false
- schema:
- type: string
- nullable: true
+ - EvaluationRules
+ parameters: []
responses:
- '200':
+ '201':
description: ''
content:
application/json:
schema:
- $ref: '#/components/schemas/ExperimentsResponse'
+ $ref: '#/components/schemas/EvaluationRule'
'400':
description: ''
content:
@@ -1715,103 +1657,54 @@ paths:
content:
application/json:
schema: {}
+ '409':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/PublicApiError'
+ '429':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/PublicApiError'
+ '500':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/PublicApiError'
security: *ref_0
- /api/public/experiment-items:
+ requestBody:
+ required: true
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/CreateEvaluationRuleRequest'
get:
- description: |-
- List experiment items with cursor-based pagination. Use this endpoint
- to export experiment item inputs, outputs, expected outputs, metadata,
- and optionally item/trace scores. Results are ordered by time
- descending.
- operationId: experiments_listItems
+ description: >-
+ List evaluation rules in newest-first creation order.
+
+
+ This includes legacy trace and dataset rules so they can be inspected
+ and migrated. Treat the cursor as opaque and return it unchanged.
+ operationId: evaluationRules_list
tags:
- - Experiments
+ - EvaluationRules
parameters:
- - name: fields
- in: query
- description: |-
- Comma-separated list of field groups to include. Available groups:
- `core`, `dataset`, `io`, `metadata`, `itemMetadata`,
- `experimentMetadata`, `scores`. If omitted, `core,dataset` is
- returned.
- required: false
- schema:
- type: string
- nullable: true
- name: limit
in: query
description: >-
- Number of experiment items to return per page. Maximum 100, default
- 50.
- required: false
- schema:
- type: integer
- nullable: true
- - name: scoreLimit
- in: query
- description: >-
- Number of scores to return per experiment item when `fields=scores`
- is requested. Maximum 50, default 50.
+ Maximum number of items to return. Defaults to `50` and cannot
+ exceed `100`.
required: false
schema:
type: integer
nullable: true
- name: cursor
in: query
- description: Versioned base64url cursor from the previous response page.
- required: false
- schema:
- type: string
- nullable: true
- - name: fromStartTime
- in: query
- description: Retrieve only experiment items started on or after this datetime.
- required: true
- schema:
- type: string
- format: date-time
- - name: toStartTime
- in: query
- description: Retrieve only experiment items started before this datetime.
- required: false
- schema:
- type: string
- format: date-time
- nullable: true
- - name: experimentId
- in: query
- description: Comma-separated list of experiment IDs.
- required: false
- schema:
- type: string
- nullable: true
- - name: experimentName
- in: query
- description: Comma-separated list of experiment names.
- required: false
- schema:
- type: string
- nullable: true
- - name: experimentItemId
- in: query
- description: Comma-separated list of experiment item IDs.
- required: false
- schema:
- type: string
- nullable: true
- - name: datasetId
- in: query
- description: Comma-separated list of dataset IDs.
- required: false
- schema:
- type: string
- nullable: true
- - name: filter
- in: query
- description: |-
- JSON string containing an array of structured filter conditions.
- Supported columns are `experimentId`, `experimentName`,
- `experimentItemId`, and `datasetId`.
+ description: Opaque cursor returned by the previous page.
required: false
schema:
type: string
@@ -1822,7 +1715,7 @@ paths:
content:
application/json:
schema:
- $ref: '#/components/schemas/ExperimentItemsResponse'
+ $ref: '#/components/schemas/EvaluationRulesPage'
'400':
description: ''
content:
@@ -1848,27 +1741,43 @@ paths:
content:
application/json:
schema: {}
+ '429':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/PublicApiError'
+ '500':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/PublicApiError'
security: *ref_0
- /api/public/feedback:
- post:
+ /api/public/v2/evaluation-rules/{evaluationRuleId}:
+ get:
description: >-
- Submit explicit user-approved feedback about Langfuse skills, MCP tools,
- CLI, docs, or public API. Do not include secrets, credentials, customer
- data, trace payloads, or unrelated use-case details.
-
-
- Not available in the HIPAA cloud region.
- operationId: feedback_submit
+ Get one evaluation rule, including a legacy trace or dataset rule, by
+ its stable identifier.
+ operationId: evaluationRules_get
tags:
- - Feedback
- parameters: []
+ - EvaluationRules
+ parameters:
+ - name: evaluationRuleId
+ in: path
+ description: >-
+ Stable evaluation-rule identifier returned by the evaluation-rule
+ endpoints.
+ required: true
+ schema:
+ type: string
responses:
- '201':
+ '200':
description: ''
content:
application/json:
schema:
- $ref: '#/components/schemas/SubmitFeedbackResponse'
+ $ref: '#/components/schemas/EvaluationRule'
'400':
description: ''
content:
@@ -1894,29 +1803,53 @@ paths:
content:
application/json:
schema: {}
- '409':
+ '429':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/PublicApiError'
+ '500':
description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/PublicApiError'
security: *ref_0
- requestBody:
- required: true
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/SubmitFeedbackRequest'
- /api/public/health:
- get:
- description: Check health of API and database
- operationId: health_health
+ patch:
+ description: >-
+ Update an evaluation rule by its stable identifier.
+
+
+ Provide only the fields to change. Providing `evaluatorAssignments`
+ replaces the complete assignment list. Replacing the list with an empty
+ array disables the rule. Setting `enabled=true` is rejected when the
+ resulting assignment list is empty, including when both fields are sent
+ in the same request.
+
+
+ Legacy trace and dataset rules follow the existing lifecycle
+ restrictions: they can be deactivated with `enabled=false`, but their
+ name, filters, sampling, and evaluator assignments cannot be changed.
+ operationId: evaluationRules_update
tags:
- - Health
- parameters: []
+ - EvaluationRules
+ parameters:
+ - name: evaluationRuleId
+ in: path
+ description: >-
+ Stable evaluation-rule identifier returned by the evaluation-rule
+ endpoints.
+ required: true
+ schema:
+ type: string
responses:
'200':
description: ''
content:
application/json:
schema:
- $ref: '#/components/schemas/HealthResponse'
+ $ref: '#/components/schemas/EvaluationRule'
'400':
description: ''
content:
@@ -1942,191 +1875,51 @@ paths:
content:
application/json:
schema: {}
- '503':
- description: ''
- /api/public/ingestion:
- post:
- description: >-
- **Deprecated:** On Langfuse Cloud, Langfuse v3 is deprecated and this
- endpoint will be removed on November 16, 2026. Send data via the
- OpenTelemetry endpoint at `POST /api/public/otel/v1/traces` instead, see
- the [OpenTelemetry integration
- docs](https://langfuse.com/integrations/native/opentelemetry).
- Self-hosted deployments are unaffected by this date; the endpoint
- becomes unavailable when they upgrade to Langfuse v4. See the [Langfuse
- v3 to v4 upgrade
- guide](https://langfuse.com/self-hosting/upgrade/upgrade-guides/upgrade-v3-to-v4).
-
-
- **Legacy endpoint for batch ingestion for Langfuse Observability.**
-
-
- -> Please use the OpenTelemetry endpoint (`/api/public/otel/v1/traces`).
- Learn more: https://langfuse.com/integrations/native/opentelemetry
-
-
- Within each batch, there can be multiple events.
-
- Each event has a type, an id, a timestamp, metadata and a body.
-
- Internally, we refer to this as the "event envelope" as it tells us
- something about the event but not the trace.
-
- We use the event id within this envelope to deduplicate messages to
- avoid processing the same event twice, i.e. the event id should be
- unique per request.
-
- The event.body.id is the ID of the actual trace and will be used for
- updates and will be visible within the Langfuse App.
-
- I.e. if you want to update a trace, you'd use the same body id, but
- separate event IDs.
-
-
- Notes:
-
- - Introduction to data model:
- https://langfuse.com/docs/observability/data-model
-
- - Batch sizes are limited to 3.5 MB in total. You need to adjust the
- number of events per batch accordingly.
-
- - The API does not return a 4xx status code for input errors. Instead,
- it responds with a 207 status code, which includes a list of the
- encountered errors.
- operationId: ingestion_batch
- tags:
- - Ingestion
- parameters: []
- responses:
- '207':
+ '409':
description: ''
content:
application/json:
schema:
- $ref: '#/components/schemas/IngestionResponse'
- '400':
- description: ''
- content:
- application/json:
- schema: {}
- '401':
- description: ''
- content:
- application/json:
- schema: {}
- '403':
- description: ''
- content:
- application/json:
- schema: {}
- '404':
+ $ref: '#/components/schemas/PublicApiError'
+ '429':
description: ''
content:
application/json:
- schema: {}
- '405':
+ schema:
+ $ref: '#/components/schemas/PublicApiError'
+ '500':
description: ''
content:
application/json:
- schema: {}
+ schema:
+ $ref: '#/components/schemas/PublicApiError'
security: *ref_0
requestBody:
required: true
content:
application/json:
schema:
- type: object
- properties:
- batch:
- type: array
- items:
- $ref: '#/components/schemas/IngestionEvent'
- description: >-
- Batch of tracing events to be ingested. Discriminated by
- attribute `type`.
- metadata:
- nullable: true
- description: >-
- Optional. Metadata field used by the Langfuse SDKs for
- debugging.
- required:
- - batch
- deprecated: true
- /api/public/metrics:
- get:
+ $ref: '#/components/schemas/UpdateEvaluationRuleRequest'
+ delete:
description: >-
- **Deprecated:** On Langfuse Cloud, Langfuse v3 is deprecated and this
- endpoint will be removed on November 16, 2026. Use `GET
- /api/public/v2/metrics?query=<urlencoded json query>` instead.
- Self-hosted deployments are unaffected by this date; the endpoint
- becomes unavailable when they upgrade to Langfuse v4. See the [Langfuse
- v3 to v4 upgrade
- guide](https://langfuse.com/self-hosting/upgrade/upgrade-guides/upgrade-v3-to-v4).
-
-
- Get metrics from the Langfuse project using a query object.
+ Delete an evaluation rule.
- Consider using the [v2 metrics
- endpoint](/api-reference#tag/metricsv2/GET/api/public/v2/metrics) for
- better performance.
+ This removes the live-ingestion rule only. It does not delete associated
+ evaluators or scores already produced by them.
- For more details, see the [Metrics API
- documentation](https://langfuse.com/docs/metrics/features/metrics-api).
- operationId: legacy_metricsV1_metrics
+ Legacy trace and dataset rules can also be deleted. Their evaluators and
+ previously produced scores are preserved.
+ operationId: evaluationRules_delete
tags:
- - LegacyMetricsV1
+ - EvaluationRules
parameters:
- - name: query
- in: query
+ - name: evaluationRuleId
+ in: path
description: >-
- JSON string containing the query parameters with the following
- structure:
-
- ```json
-
- {
- "view": string, // Required. One of "traces", "observations", "scores-numeric", "scores-boolean", "scores-categorical"
- "dimensions": [ // Optional. Default: []
- {
- "field": string // Field to group by, e.g. "name", "userId", "sessionId"
- }
- ],
- "metrics": [ // Required. At least one metric must be provided
- {
- "measure": string, // What to measure, e.g. "count", "latency", "value"
- "aggregation": string // How to aggregate, e.g. "count", "sum", "avg", "p95", "histogram"
- }
- ],
- "filters": [ // Optional. Default: []
- {
- "column": string, // Column to filter on
- "operator": string, // Operator, e.g. "=", ">", "<", "contains"
- "value": any, // Value to compare against
- "type": string, // Data type, e.g. "string", "number", "stringObject"
- "key": string // Required only when filtering on metadata
- }
- ],
- "timeDimension": { // Optional. Default: null. If provided, results will be grouped by time
- "granularity": string // One of "minute", "hour", "day", "week", "month", "auto"
- },
- "fromTimestamp": string, // Required. ISO datetime string for start of time range
- "toTimestamp": string, // Required. ISO datetime string for end of time range
- "orderBy": [ // Optional. Default: null
- {
- "field": string, // Field to order by
- "direction": string // "asc" or "desc"
- }
- ],
- "config": { // Optional. Query-specific configuration
- "bins": number, // Optional. Number of bins for histogram (1-100), default: 10
- "row_limit": number // Optional. Row limit for results (1-1000)
- }
- }
-
- ```
+ Stable evaluation-rule identifier returned by the evaluation-rule
+ endpoints.
required: true
schema:
type: string
@@ -2136,7 +1929,7 @@ paths:
content:
application/json:
schema:
- $ref: '#/components/schemas/legacyMetricsResponse'
+ $ref: '#/components/schemas/DeletedEvaluationRule'
'400':
description: ''
content:
@@ -2162,40 +1955,44 @@ paths:
content:
application/json:
schema: {}
+ '429':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/PublicApiError'
+ '500':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/PublicApiError'
security: *ref_0
- deprecated: true
- /api/public/observations/{observationId}:
- get:
+ /api/public/v2/evaluators:
+ post:
description: >-
- **Deprecated:** On Langfuse Cloud, Langfuse v3 is deprecated and this
- endpoint will be removed on November 16, 2026. Use `GET
- /api/public/v2/observations?fromStartTime=<from>&toStartTime=<to>`
- instead. Self-hosted deployments are unaffected by this date; the
- endpoint becomes unavailable when they upgrade to Langfuse v4. See the
- [Langfuse v3 to v4 upgrade
- guide](https://langfuse.com/self-hosting/upgrade/upgrade-guides/upgrade-v3-to-v4).
+ Create an evaluator in the authenticated project.
- Get a observation
- operationId: legacy_observationsV1_get
+ An evaluator defines **how** Langfuse should score data. LLM-as-a-judge
+ evaluators define a prompt, expected structured output, and optional
+ model configuration. Code evaluators define source code and a runtime
+ language.
+
+
+ This always creates a new evaluator with version `1`. Names are not
+ identifiers and do not need to be unique.
+ operationId: evaluators_create
tags:
- - LegacyObservationsV1
- parameters:
- - name: observationId
- in: path
- description: >-
- The unique langfuse identifier of an observation, can be an event,
- span or generation
- required: true
- schema:
- type: string
+ - Evaluators
+ parameters: []
responses:
- '200':
+ '201':
description: ''
content:
application/json:
schema:
- $ref: '#/components/schemas/ObservationsViewSingle'
+ $ref: '#/components/schemas/Evaluator'
'400':
description: ''
content:
@@ -2221,268 +2018,55 @@ paths:
content:
application/json:
schema: {}
+ '412':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/PublicApiError'
+ '429':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/PublicApiError'
+ '500':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/PublicApiError'
security: *ref_0
- deprecated: true
- /api/public/observations:
+ requestBody:
+ required: true
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/CreateEvaluatorRequest'
get:
description: >-
- **Deprecated:** On Langfuse Cloud, Langfuse v3 is deprecated and this
- endpoint will be removed on November 16, 2026. Use `GET
- /api/public/v2/observations?fromStartTime=<from>&toStartTime=<to>`
- instead. Self-hosted deployments are unaffected by this date; the
- endpoint becomes unavailable when they upgrade to Langfuse v4. See the
- [Langfuse v3 to v4 upgrade
- guide](https://langfuse.com/self-hosting/upgrade/upgrade-guides/upgrade-v3-to-v4).
+ List evaluators in newest-first creation order.
- Get a list of observations.
-
-
- Consider using the [v2 observations
- endpoint](/api-reference#tag/observationsv2/GET/api/public/v2/observations)
- for cursor-based pagination and field selection.
- operationId: legacy_observationsV1_getMany
+ Every evaluator includes its latest definition and version metadata
+ flattened into the evaluator object, plus associated evaluation rules.
+ Treat the cursor as opaque and return it unchanged.
+ operationId: evaluators_list
tags:
- - LegacyObservationsV1
+ - Evaluators
parameters:
- - name: page
- in: query
- description: Page number, starts at 1.
- required: false
- schema:
- type: integer
- nullable: true
- name: limit
in: query
description: >-
- Limit of items per page. If you encounter api issues due to too
- large page sizes, try to reduce the limit.
+ Maximum number of items to return. Defaults to `50` and cannot
+ exceed `100`.
required: false
schema:
type: integer
nullable: true
- - name: name
- in: query
- required: false
- schema:
- type: string
- nullable: true
- - name: userId
- in: query
- required: false
- schema:
- type: string
- nullable: true
- - name: type
- in: query
- required: false
- schema:
- type: string
- nullable: true
- - name: traceId
- in: query
- required: false
- schema:
- type: string
- nullable: true
- - name: level
- in: query
- description: >-
- Optional filter for observations with a specific level (e.g.
- "DEBUG", "DEFAULT", "WARNING", "ERROR").
- required: false
- schema:
- $ref: '#/components/schemas/ObservationLevel'
- nullable: true
- - name: parentObservationId
- in: query
- required: false
- schema:
- type: string
- nullable: true
- - name: environment
- in: query
- description: >-
- Optional filter for observations where the environment is one of the
- provided values.
- required: false
- schema:
- type: array
- items:
- type: string
- nullable: true
- - name: fromStartTime
- in: query
- description: >-
- Retrieve only observations with a start_time on or after this
- datetime (ISO 8601).
- required: false
- schema:
- type: string
- format: date-time
- nullable: true
- - name: toStartTime
- in: query
- description: >-
- Retrieve only observations with a start_time before this datetime
- (ISO 8601).
- required: false
- schema:
- type: string
- format: date-time
- nullable: true
- - name: version
- in: query
- description: Optional filter to only include observations with a certain version.
- required: false
- schema:
- type: string
- nullable: true
- - name: filter
+ - name: cursor
in: query
- description: >-
- JSON string containing an array of filter conditions. When provided,
- this takes precedence over query parameter filters (userId, name,
- type, level, environment, fromStartTime, ...).
-
-
- ## Filter Structure
-
- Each filter condition has the following structure:
-
- ```json
-
- [
- {
- "type": string, // Required. One of: "datetime", "string", "number", "stringOptions", "categoryOptions", "arrayOptions", "stringObject", "numberObject", "boolean", "null"
- "column": string, // Required. Column to filter on (see available columns below)
- "operator": string, // Required. Operator based on type:
- // - datetime: ">", "<", ">=", "<="
- // - string: "=", "contains", "does not contain", "starts with", "ends with"
- // - stringOptions: "any of", "none of"
- // - categoryOptions: "any of", "none of"
- // - arrayOptions: "any of", "none of", "all of"
- // - number: "=", ">", "<", ">=", "<="
- // - stringObject: "=", "contains", "does not contain", "starts with", "ends with"
- // - numberObject: "=", ">", "<", ">=", "<="
- // - boolean: "=", "<>"
- // - null: "is null", "is not null"
- "value": any, // Required (except for null type). Value to compare against. Type depends on filter type
- "key": string // Required only for stringObject, numberObject, and categoryOptions types when filtering on nested fields like metadata
- }
- ]
-
- ```
-
-
- ## Available Columns
-
-
- ### Core Observation Fields
-
- - `id` (string) - Observation ID
-
- - `type` (string) - Observation type (SPAN, GENERATION, EVENT)
-
- - `name` (string) - Observation name
-
- - `traceId` (string) - Associated trace ID
-
- - `startTime` (datetime) - Observation start time
-
- - `endTime` (datetime) - Observation end time
-
- - `environment` (string) - Environment tag
-
- - `level` (string) - Log level (DEBUG, DEFAULT, WARNING, ERROR)
-
- - `statusMessage` (string) - Status message
-
- - `version` (string) - Version tag
-
-
- ### Performance Metrics
-
- - `latency` (number) - Latency in seconds (calculated: end_time -
- start_time)
-
- - `timeToFirstToken` (number) - Time to first token in seconds
-
- - `tokensPerSecond` (number) - Output tokens per second
-
-
- ### Token Usage
-
- - `inputTokens` (number) - Number of input tokens
-
- - `outputTokens` (number) - Number of output tokens
-
- - `totalTokens` (number) - Total tokens (alias: `tokens`)
-
-
- ### Cost Metrics
-
- - `inputCost` (number) - Input cost in USD
-
- - `outputCost` (number) - Output cost in USD
-
- - `totalCost` (number) - Total cost in USD
-
-
- ### Model Information
-
- - `model` (string) - Provided model name
-
- - `promptName` (string) - Associated prompt name
-
- - `promptVersion` (number) - Associated prompt version
-
-
- ### Structured Data
-
- - `metadata` (stringObject/numberObject/categoryOptions) - Metadata
- key-value pairs. Use `key` parameter to filter on specific metadata
- keys.
-
-
- ### Associated Trace Fields (requires join with traces table)
-
- - `userId` (string) - User ID from associated trace
-
- - `traceName` (string) - Name from associated trace
-
- - `traceEnvironment` (string) - Environment from associated trace
-
- - `traceTags` (arrayOptions) - Tags from associated trace
-
-
- ## Filter Examples
-
- ```json
-
- [
- {
- "type": "string",
- "column": "type",
- "operator": "=",
- "value": "GENERATION"
- },
- {
- "type": "number",
- "column": "latency",
- "operator": ">=",
- "value": 2.5
- },
- {
- "type": "stringObject",
- "column": "metadata",
- "key": "environment",
- "operator": "=",
- "value": "production"
- }
- ]
-
- ```
+ description: Opaque cursor returned by the previous page.
required: false
schema:
type: string
@@ -2493,7 +2077,7 @@ paths:
content:
application/json:
schema:
- $ref: '#/components/schemas/legacyObservationsViews'
+ $ref: '#/components/schemas/EvaluatorsPage'
'400':
description: ''
content:
@@ -2519,24 +2103,46 @@ paths:
content:
application/json:
schema: {}
+ '429':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/PublicApiError'
+ '500':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/PublicApiError'
security: *ref_0
- deprecated: true
- /api/public/scores/{scoreId}:
- delete:
- description: Delete a score (supports both trace and session scores)
- operationId: legacy_scoreV1_delete
+ /api/public/v2/evaluators/{evaluatorId}:
+ get:
+ description: >-
+ Get one evaluator by its stable identifier.
+
+
+ The response includes the evaluator's latest definition and version
+ metadata flattened into the evaluator object, plus associated evaluation
+ rules. Use the version-history endpoint when older definitions are
+ needed.
+ operationId: evaluators_get
tags:
- - LegacyScoreV1
+ - Evaluators
parameters:
- - name: scoreId
+ - name: evaluatorId
in: path
- description: The unique langfuse identifier of a score
+ description: Stable evaluator identifier returned by the evaluator endpoints.
required: true
schema:
type: string
responses:
- '204':
+ '200':
description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/Evaluator'
'400':
description: ''
content:
@@ -2562,35 +2168,58 @@ paths:
content:
application/json:
schema: {}
+ '429':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/PublicApiError'
+ '500':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/PublicApiError'
security: *ref_0
- /api/public/llm-connections:
- get:
- description: Get all LLM connections in a project
- operationId: llmConnections_list
+ patch:
+ description: >-
+ Update an evaluator by its stable identifier.
+
+
+ Provide only the top-level fields to change. Metadata-only changes do
+ not create a version. Evaluator type cannot change.
+
+
+ Definition fields are flattened into the request. To replace a
+ definition, include `type` and every definition field for that type.
+ Definition fields are replaced as a complete unit rather than merged.
+ For LLM-as-a-judge evaluators, omitting or setting `modelConfig` to
+ `null` selects the project's default evaluation model.
+
+
+ Replacing a definition automatically returns an evaluator paused by an
+ invalid or missing model configuration to `active`, while preserving its
+ id and evaluation-rule assignments. Pauses caused by provider
+ authentication, billing, connectivity, account state, or an unknown
+ legacy reason remain paused until the explicit reactivation check
+ succeeds.
+ operationId: evaluators_update
tags:
- - LlmConnections
+ - Evaluators
parameters:
- - name: page
- in: query
- description: page number, starts at 1
- required: false
- schema:
- type: integer
- nullable: true
- - name: limit
- in: query
- description: limit of items per page
- required: false
+ - name: evaluatorId
+ in: path
+ description: Stable evaluator identifier returned by the evaluator endpoints.
+ required: true
schema:
- type: integer
- nullable: true
+ type: string
responses:
'200':
description: ''
content:
application/json:
schema:
- $ref: '#/components/schemas/PaginatedLlmConnections'
+ $ref: '#/components/schemas/Evaluator'
'400':
description: ''
content:
@@ -2616,65 +2245,51 @@ paths:
content:
application/json:
schema: {}
- security: *ref_0
- put:
- description: >-
- Create or update an LLM connection. The connection is upserted on
- provider.
- operationId: llmConnections_upsert
- tags:
- - LlmConnections
- parameters: []
- responses:
- '200':
+ '409':
description: ''
content:
application/json:
schema:
- $ref: '#/components/schemas/LlmConnection'
- '400':
- description: ''
- content:
- application/json:
- schema: {}
- '401':
- description: ''
- content:
- application/json:
- schema: {}
- '403':
+ $ref: '#/components/schemas/PublicApiError'
+ '412':
description: ''
content:
application/json:
- schema: {}
- '404':
+ schema:
+ $ref: '#/components/schemas/PublicApiError'
+ '429':
description: ''
content:
application/json:
- schema: {}
- '405':
+ schema:
+ $ref: '#/components/schemas/PublicApiError'
+ '500':
description: ''
content:
application/json:
- schema: {}
+ schema:
+ $ref: '#/components/schemas/PublicApiError'
security: *ref_0
requestBody:
required: true
content:
application/json:
schema:
- $ref: '#/components/schemas/UpsertLlmConnectionRequest'
- /api/public/llm-connections/{id}:
+ $ref: '#/components/schemas/UpdateEvaluatorRequest'
delete:
description: >-
- Delete an LLM connection by id. Evaluators that depend on the deleted
- connection are automatically paused.
- operationId: llmConnections_delete
+ Delete an evaluator and all of its stored versions.
+
+
+ Associated evaluation-rule assignments are also removed. Scores already
+ produced by the evaluator are preserved.
+ operationId: evaluators_delete
tags:
- - LlmConnections
+ - Evaluators
parameters:
- - name: id
+ - name: evaluatorId
in: path
+ description: Stable evaluator identifier returned by the evaluator endpoints.
required: true
schema:
type: string
@@ -2684,7 +2299,7 @@ paths:
content:
application/json:
schema:
- $ref: '#/components/schemas/DeleteLlmConnectionResponse'
+ $ref: '#/components/schemas/DeletedEvaluator'
'400':
description: ''
content:
@@ -2710,27 +2325,61 @@ paths:
content:
application/json:
schema: {}
+ '429':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/PublicApiError'
+ '500':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/PublicApiError'
security: *ref_0
- /api/public/media/{mediaId}:
+ /api/public/v2/evaluators/{evaluatorId}/versions:
get:
- description: Get a media record
- operationId: media_get
+ description: >-
+ List an evaluator's version history in newest-first order.
+
+
+ This endpoint is intended for history and audit use cases. Ordinary
+ clients can use the flattened `version` and definition fields on the
+ evaluator response.
+ operationId: evaluators_listVersions
tags:
- - Media
+ - Evaluators
parameters:
- - name: mediaId
+ - name: evaluatorId
in: path
- description: The unique langfuse identifier of a media record
+ description: Stable evaluator identifier returned by the evaluator endpoints.
required: true
schema:
type: string
+ - name: limit
+ in: query
+ description: >-
+ Maximum number of versions to return. Defaults to `50` and cannot
+ exceed `100`.
+ required: false
+ schema:
+ type: integer
+ nullable: true
+ - name: cursor
+ in: query
+ description: Opaque cursor returned by the previous page.
+ required: false
+ schema:
+ type: string
+ nullable: true
responses:
'200':
description: ''
content:
application/json:
schema:
- $ref: '#/components/schemas/GetMediaResponse'
+ $ref: '#/components/schemas/EvaluatorVersionsPage'
'400':
description: ''
content:
@@ -2756,22 +2405,112 @@ paths:
content:
application/json:
schema: {}
+ '429':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/PublicApiError'
+ '500':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/PublicApiError'
security: *ref_0
- patch:
- description: Patch a media record
- operationId: media_patch
+ /api/public/experiments:
+ get:
+ description: |-
+ List experiments with cursor-based pagination. Results are ordered by
+ latest experiment activity descending.
+ operationId: experiments_list
tags:
- - Media
+ - Experiments
parameters:
- - name: mediaId
- in: path
- description: The unique langfuse identifier of a media record
+ - name: fields
+ in: query
+ description: |-
+ Comma-separated list of field groups to include. Available groups:
+ `core`, `metadata`, `scores`. If omitted, `core` is returned.
+ required: false
+ schema:
+ type: string
+ nullable: true
+ - name: limit
+ in: query
+ description: Number of experiments to return per page. Maximum 100, default 50.
+ required: false
+ schema:
+ type: integer
+ nullable: true
+ - name: scoreLimit
+ in: query
+ description: >-
+ Number of scores to return per experiment when `fields=scores` is
+ requested. Maximum 50, default 50.
+ required: false
+ schema:
+ type: integer
+ nullable: true
+ - name: cursor
+ in: query
+ description: Versioned base64url cursor from the previous response page.
+ required: false
+ schema:
+ type: string
+ nullable: true
+ - name: fromStartTime
+ in: query
+ description: Retrieve only experiments on or after this datetime.
required: true
schema:
type: string
+ format: date-time
+ - name: toStartTime
+ in: query
+ description: Retrieve only experiments before this datetime.
+ required: false
+ schema:
+ type: string
+ format: date-time
+ nullable: true
+ - name: id
+ in: query
+ description: Comma-separated list of experiment IDs.
+ required: false
+ schema:
+ type: string
+ nullable: true
+ - name: name
+ in: query
+ description: Comma-separated list of experiment names.
+ required: false
+ schema:
+ type: string
+ nullable: true
+ - name: datasetId
+ in: query
+ description: Comma-separated list of dataset IDs.
+ required: false
+ schema:
+ type: string
+ nullable: true
+ - name: filter
+ in: query
+ description: |-
+ JSON string containing an array of structured filter conditions.
+ Supported columns are `id`, `name`, and `datasetId`.
+ required: false
+ schema:
+ type: string
+ nullable: true
responses:
- '204':
+ '200':
description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/ExperimentsResponse'
'400':
description: ''
content:
@@ -2798,26 +2537,159 @@ paths:
application/json:
schema: {}
security: *ref_0
- requestBody:
- required: true
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/PatchMediaBody'
- /api/public/media:
- post:
- description: Get a presigned upload URL for a media record
- operationId: media_getUploadUrl
- tags:
- - Media
- parameters: []
- responses:
- '200':
- description: ''
+ /api/public/experiment-items:
+ get:
+ description: |-
+ List experiment items with cursor-based pagination. Use this endpoint
+ to export experiment item inputs, outputs, expected outputs, metadata,
+ and optionally item/trace scores. Results are ordered by time
+ descending.
+ operationId: experiments_listItems
+ tags:
+ - Experiments
+ parameters:
+ - name: fields
+ in: query
+ description: |-
+ Comma-separated list of field groups to include. Available groups:
+ `core`, `dataset`, `io`, `metadata`, `itemMetadata`,
+ `experimentMetadata`, `scores`. If omitted, `core,dataset` is
+ returned.
+ required: false
+ schema:
+ type: string
+ nullable: true
+ - name: limit
+ in: query
+ description: >-
+ Number of experiment items to return per page. Maximum 100, default
+ 50.
+ required: false
+ schema:
+ type: integer
+ nullable: true
+ - name: scoreLimit
+ in: query
+ description: >-
+ Number of scores to return per experiment item when `fields=scores`
+ is requested. Maximum 50, default 50.
+ required: false
+ schema:
+ type: integer
+ nullable: true
+ - name: cursor
+ in: query
+ description: Versioned base64url cursor from the previous response page.
+ required: false
+ schema:
+ type: string
+ nullable: true
+ - name: fromStartTime
+ in: query
+ description: Retrieve only experiment items started on or after this datetime.
+ required: true
+ schema:
+ type: string
+ format: date-time
+ - name: toStartTime
+ in: query
+ description: Retrieve only experiment items started before this datetime.
+ required: false
+ schema:
+ type: string
+ format: date-time
+ nullable: true
+ - name: experimentId
+ in: query
+ description: Comma-separated list of experiment IDs.
+ required: false
+ schema:
+ type: string
+ nullable: true
+ - name: experimentName
+ in: query
+ description: Comma-separated list of experiment names.
+ required: false
+ schema:
+ type: string
+ nullable: true
+ - name: experimentItemId
+ in: query
+ description: Comma-separated list of experiment item IDs.
+ required: false
+ schema:
+ type: string
+ nullable: true
+ - name: datasetId
+ in: query
+ description: Comma-separated list of dataset IDs.
+ required: false
+ schema:
+ type: string
+ nullable: true
+ - name: filter
+ in: query
+ description: |-
+ JSON string containing an array of structured filter conditions.
+ Supported columns are `experimentId`, `experimentName`,
+ `experimentItemId`, and `datasetId`.
+ required: false
+ schema:
+ type: string
+ nullable: true
+ responses:
+ '200':
+ description: ''
content:
application/json:
schema:
- $ref: '#/components/schemas/GetMediaUploadUrlResponse'
+ $ref: '#/components/schemas/ExperimentItemsResponse'
+ '400':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '401':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '403':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '404':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '405':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ security: *ref_0
+ /api/public/feedback:
+ post:
+ description: >-
+ Submit explicit user-approved feedback about Langfuse skills, MCP tools,
+ CLI, docs, or public API. Do not include secrets, credentials, customer
+ data, trace payloads, or unrelated use-case details.
+
+
+ Not available in the HIPAA cloud region.
+ operationId: feedback_submit
+ tags:
+ - Feedback
+ parameters: []
+ responses:
+ '201':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/SubmitFeedbackResponse'
'400':
description: ''
content:
@@ -2843,243 +2715,190 @@ paths:
content:
application/json:
schema: {}
+ '409':
+ description: ''
security: *ref_0
requestBody:
required: true
content:
application/json:
schema:
- $ref: '#/components/schemas/GetMediaUploadUrlRequest'
- /api/public/v2/metrics:
+ $ref: '#/components/schemas/SubmitFeedbackRequest'
+ /api/public/health:
get:
+ description: Check health of API and database
+ operationId: health_health
+ tags:
+ - Health
+ parameters: []
+ responses:
+ '200':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/HealthResponse'
+ '400':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '401':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '403':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '404':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '405':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '503':
+ description: ''
+ /api/public/ingestion:
+ post:
description: >-
- Get metrics from the Langfuse project using a query object. V2 endpoint
- with optimized performance.
-
-
- ## V2 Differences
-
- - Supports `observations`, `scores-numeric`, `scores-boolean`, and
- `scores-categorical` views only (traces view not supported)
-
- - Direct access to tags and release fields on observations
-
- - Semantic-root filtering and grouping through the v2-only
- `isRootObservation` dimension
-
- - Backwards-compatible: traceName, traceRelease, traceVersion dimensions
- are still available on observations view
+ **Deprecated:** On Langfuse Cloud, Langfuse v3 is deprecated and this
+ endpoint will be removed on November 16, 2026. Send data via the
+ OpenTelemetry endpoint at `POST /api/public/otel/v1/traces` instead, see
+ the [OpenTelemetry integration
+ docs](https://langfuse.com/integrations/native/opentelemetry).
+ Self-hosted deployments are unaffected by this date; the endpoint
+ becomes unavailable when they upgrade to Langfuse v4. See the [Langfuse
+ v3 to v4 upgrade
+ guide](https://langfuse.com/self-hosting/upgrade/upgrade-guides/upgrade-v3-to-v4).
- - High cardinality dimensions are not supported and will return a 400
- error (see below)
+ **Legacy endpoint for batch ingestion for Langfuse Observability.**
- For more details, see the [Metrics API
- documentation](https://langfuse.com/docs/metrics/features/metrics-api).
+ -> Please use the OpenTelemetry endpoint (`/api/public/otel/v1/traces`).
+ Learn more: https://langfuse.com/integrations/native/opentelemetry
- ## Available Views
+ Within each batch, there can be multiple events.
- ### observations
+ Each event has a type, an id, a timestamp, metadata and a body.
- Query observation-level data (spans, generations, events).
+ Internally, we refer to this as the "event envelope" as it tells us
+ something about the event but not the trace.
+ We use the event id within this envelope to deduplicate messages to
+ avoid processing the same event twice, i.e. the event id should be
+ unique per request.
- **Dimensions:**
+ The event.body.id is the ID of the actual trace and will be used for
+ updates and will be visible within the Langfuse App.
- - `environment` - Deployment environment (e.g., production, staging)
+ I.e. if you want to update a trace, you'd use the same body id, but
+ separate event IDs.
- - `type` - Type of observation (SPAN, GENERATION, EVENT)
- - `name` - Name of the observation
+ Notes:
- - `level` - Logging level of the observation
+ - Introduction to data model:
+ https://langfuse.com/docs/observability/data-model
- - `version` - Version of the observation
-
- - `tags` - User-defined tags
-
- - `release` - Release version
-
- - `traceName` - Name of the parent trace (backwards-compatible)
-
- - `traceRelease` - Release version of the parent trace
- (backwards-compatible, maps to release)
-
- - `traceVersion` - Version of the parent trace (backwards-compatible,
- maps to version)
-
- - `providedModelName` - Name of the model used
-
- - `promptName` - Name of the prompt used
-
- - `promptVersion` - Version of the prompt used
-
- - `isRootObservation` - Boolean semantic-root status. `true` includes
- physical roots and app roots whose SDK parent is external (so
- `parentObservationId` may be non-null).
-
- - `startTimeMonth` - Month of start_time in YYYY-MM format
-
-
- **Measures:**
-
- - `count` - Total number of observations
-
- - `latency` - Observation latency (milliseconds)
-
- - `streamingLatency` - Generation latency from completion start to end
- (milliseconds)
-
- - `inputTokens` - Sum of input tokens consumed
-
- - `outputTokens` - Sum of output tokens produced
-
- - `totalTokens` - Sum of all tokens consumed
-
- - `outputTokensPerSecond` - Output tokens per second
-
- - `tokensPerSecond` - Total tokens per second
-
- - `inputCost` - Input cost (USD)
-
- - `outputCost` - Output cost (USD)
-
- - `totalCost` - Total cost (USD)
-
- - `timeToFirstToken` - Time to first token (milliseconds)
-
- - `countScores` - Number of scores attached to the observation
-
-
- ### scores-numeric
-
- Query numeric and boolean score data.
-
-
- **Dimensions:**
-
- - `environment` - Deployment environment
-
- - `name` - Name of the score (e.g., accuracy, toxicity)
-
- - `source` - Origin of the score (API, ANNOTATION, EVAL)
-
- - `dataType` - Data type (NUMERIC, BOOLEAN)
-
- - `configId` - Identifier of the score config
-
- - `timestampMonth` - Month in YYYY-MM format
-
- - `timestampDay` - Day in YYYY-MM-DD format
-
- - `value` - Numeric value of the score
-
- - `traceName` - Name of the parent trace
-
- - `tags` - Tags
-
- - `traceRelease` - Release version
-
- - `traceVersion` - Version
-
- - `observationName` - Name of the associated observation
-
- - `observationModelName` - Model name of the associated observation
-
- - `observationPromptName` - Prompt name of the associated observation
-
- - `observationPromptVersion` - Prompt version of the associated
- observation
-
-
- **Measures:**
-
- - `count` - Total number of scores
-
- - `value` - Score value (for aggregations)
-
-
- ### scores-boolean
-
- Query boolean score data. It has the same score and parent
- trace/observation dimensions as scores-numeric, plus:
-
-
- **Dimensions:**
-
- - `booleanValue` - Boolean value for true/false grouping and filtering
-
-
- **Measures:**
-
- - `count` - Total number of boolean scores
-
- - `value` - Numeric 0/1 score value; `avg` returns the true-rate
-
-
- ### scores-categorical
-
- Query categorical score data. Same dimensions as scores-numeric except
- uses `stringValue` instead of `value`.
-
-
- **Measures:**
-
- - `count` - Total number of scores
-
-
- ## High Cardinality Dimensions
-
- The following dimensions cannot be used as grouping dimensions in v2
- metrics API as they can cause performance issues.
-
- Use them in filters instead.
-
-
- **observations view:**
-
- - `id` - Use traceId filter to narrow down results
-
- - `traceId` - Use traceId filter instead
-
- - `userId` - Use userId filter instead
-
- - `sessionId` - Use sessionId filter instead
-
- - `parentObservationId` - Use parentObservationId filter instead
-
-
- **scores-numeric / scores-boolean / scores-categorical views:**
-
- - `id` - Use specific filters to narrow down results
-
- - `traceId` - Use traceId filter instead
-
- - `userId` - Use userId filter instead
-
- - `sessionId` - Use sessionId filter instead
-
- - `observationId` - Use observationId filter instead
+ - Batch sizes are limited to 3.5 MB in total. You need to adjust the
+ number of events per batch accordingly.
+ - The API does not return a 4xx status code for input errors. Instead,
+ it responds with a 207 status code, which includes a list of the
+ encountered errors.
+ operationId: ingestion_batch
+ tags:
+ - Ingestion
+ parameters: []
+ responses:
+ '207':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/IngestionResponse'
+ '400':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '401':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '403':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '404':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '405':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ security: *ref_0
+ requestBody:
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ properties:
+ batch:
+ type: array
+ items:
+ $ref: '#/components/schemas/IngestionEvent'
+ description: >-
+ Batch of tracing events to be ingested. Discriminated by
+ attribute `type`.
+ metadata:
+ nullable: true
+ description: >-
+ Optional. Metadata field used by the Langfuse SDKs for
+ debugging.
+ required:
+ - batch
+ deprecated: true
+ /api/public/metrics:
+ get:
+ description: >-
+ **Deprecated:** On Langfuse Cloud, Langfuse v3 is deprecated and this
+ endpoint will be removed on November 16, 2026. Use `GET
+ /api/public/v2/metrics?query=<urlencoded json query>` instead.
+ Self-hosted deployments are unaffected by this date; the endpoint
+ becomes unavailable when they upgrade to Langfuse v4. See the [Langfuse
+ v3 to v4 upgrade
+ guide](https://langfuse.com/self-hosting/upgrade/upgrade-guides/upgrade-v3-to-v4).
- ## Aggregations
- Available aggregation functions: `sum`, `avg`, `count`, `max`, `min`,
- `p50`, `p75`, `p90`, `p95`, `p99`, `histogram`
+ Get metrics from the Langfuse project using a query object.
- ## Time Granularities
+ Consider using the [v2 metrics
+ endpoint](/api-reference#tag/metricsv2/GET/api/public/v2/metrics) for
+ better performance.
- Available granularities for timeDimension: `auto`, `minute`, `hour`,
- `day`, `week`, `month`
- - `auto` bins the data into approximately 50 buckets based on the time
- range
- operationId: metrics_metrics
+ For more details, see the [Metrics API
+ documentation](https://langfuse.com/docs/metrics/features/metrics-api).
+ operationId: legacy_metricsV1_metrics
tags:
- - Metrics
+ - LegacyMetricsV1
parameters:
- name: query
in: query
@@ -3090,69 +2909,45 @@ paths:
```json
{
- "view": string, // Required. One of "observations", "scores-numeric", "scores-boolean", "scores-categorical"
+ "view": string, // Required. One of "traces", "observations", "scores-numeric", "scores-boolean", "scores-categorical"
"dimensions": [ // Optional. Default: []
{
- "field": string // Field to group by (see available dimensions above)
+ "field": string // Field to group by, e.g. "name", "userId", "sessionId"
}
],
"metrics": [ // Required. At least one metric must be provided
{
- "measure": string, // What to measure (see available measures above)
- "aggregation": string // How to aggregate: "sum", "avg", "count", "max", "min", "p50", "p75", "p90", "p95", "p99", "histogram"
+ "measure": string, // What to measure, e.g. "count", "latency", "value"
+ "aggregation": string // How to aggregate, e.g. "count", "sum", "avg", "p95", "histogram"
}
],
"filters": [ // Optional. Default: []
{
- "column": string, // Column to filter on (any dimension field)
- "operator": string, // Operator based on type:
- // - datetime: ">", "<", ">=", "<="
- // - string: "=", "contains", "does not contain", "starts with", "ends with"
- // - stringOptions: "any of", "none of"
- // - arrayOptions: "any of", "none of", "all of"
- // - number: "=", ">", "<", ">=", "<="
- // - stringObject/numberObject: same as string/number with required "key"
- // - boolean: "=", "<>"
- // - null: "is null", "is not null"
+ "column": string, // Column to filter on
+ "operator": string, // Operator, e.g. "=", ">", "<", "contains"
"value": any, // Value to compare against
- "type": string, // Data type: "datetime", "string", "number", "stringOptions", "categoryOptions", "arrayOptions", "stringObject", "numberObject", "boolean", "null"
- "key": string // Required only for stringObject/numberObject types (e.g., metadata filtering)
+ "type": string, // Data type, e.g. "string", "number", "stringObject"
+ "key": string // Required only when filtering on metadata
}
],
"timeDimension": { // Optional. Default: null. If provided, results will be grouped by time
- "granularity": string // One of "auto", "minute", "hour", "day", "week", "month"
+ "granularity": string // One of "minute", "hour", "day", "week", "month", "auto"
},
"fromTimestamp": string, // Required. ISO datetime string for start of time range
- "toTimestamp": string, // Required. ISO datetime string for end of time range (must be after fromTimestamp)
+ "toTimestamp": string, // Required. ISO datetime string for end of time range
"orderBy": [ // Optional. Default: null
{
- "field": string, // Field to order by (dimension or metric alias)
+ "field": string, // Field to order by
"direction": string // "asc" or "desc"
}
],
"config": { // Optional. Query-specific configuration
- "bins": number, // Optional. Number of bins for histogram aggregation (1-100), default: 10
- "row_limit": number // Optional. Maximum number of rows to return (1-1000), default: 100
+ "bins": number, // Optional. Number of bins for histogram (1-100), default: 10
+ "row_limit": number // Optional. Row limit for results (1-1000)
}
}
```
-
-
- For example, to count semantic roots (including app roots with a
- non-null external parent), use a boolean filter:
-
- ```json
-
- {
- "view": "observations",
- "metrics": [{"measure": "count", "aggregation": "count"}],
- "filters": [{"column": "isRootObservation", "operator": "=", "value": true, "type": "boolean"}],
- "fromTimestamp": "2025-01-01T00:00:00.000Z",
- "toTimestamp": "2025-02-01T00:00:00.000Z"
- }
-
- ```
required: true
schema:
type: string
@@ -3162,7 +2957,7 @@ paths:
content:
application/json:
schema:
- $ref: '#/components/schemas/MetricsV2Response'
+ $ref: '#/components/schemas/legacyMetricsResponse'
'400':
description: ''
content:
@@ -3189,20 +2984,39 @@ paths:
application/json:
schema: {}
security: *ref_0
- /api/public/models:
- post:
- description: Create a model
- operationId: models_create
+ deprecated: true
+ /api/public/observations/{observationId}:
+ get:
+ description: >-
+ **Deprecated:** On Langfuse Cloud, Langfuse v3 is deprecated and this
+ endpoint will be removed on November 16, 2026. Use `GET
+ /api/public/v2/observations?fromStartTime=<from>&toStartTime=<to>`
+ instead. Self-hosted deployments are unaffected by this date; the
+ endpoint becomes unavailable when they upgrade to Langfuse v4. See the
+ [Langfuse v3 to v4 upgrade
+ guide](https://langfuse.com/self-hosting/upgrade/upgrade-guides/upgrade-v3-to-v4).
+
+
+ Get a observation
+ operationId: legacy_observationsV1_get
tags:
- - Models
- parameters: []
+ - LegacyObservationsV1
+ parameters:
+ - name: observationId
+ in: path
+ description: >-
+ The unique langfuse identifier of an observation, can be an event,
+ span or generation
+ required: true
+ schema:
+ type: string
responses:
'200':
description: ''
content:
application/json:
schema:
- $ref: '#/components/schemas/Model'
+ $ref: '#/components/schemas/ObservationsViewSingle'
'400':
description: ''
content:
@@ -3229,343 +3043,59 @@ paths:
application/json:
schema: {}
security: *ref_0
- requestBody:
- required: true
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/CreateModelRequest'
+ deprecated: true
+ /api/public/observations:
get:
- description: Get all models
- operationId: models_list
+ description: >-
+ **Deprecated:** On Langfuse Cloud, Langfuse v3 is deprecated and this
+ endpoint will be removed on November 16, 2026. Use `GET
+ /api/public/v2/observations?fromStartTime=<from>&toStartTime=<to>`
+ instead. Self-hosted deployments are unaffected by this date; the
+ endpoint becomes unavailable when they upgrade to Langfuse v4. See the
+ [Langfuse v3 to v4 upgrade
+ guide](https://langfuse.com/self-hosting/upgrade/upgrade-guides/upgrade-v3-to-v4).
+
+
+ Get a list of observations.
+
+
+ Consider using the [v2 observations
+ endpoint](/api-reference#tag/observationsv2/GET/api/public/v2/observations)
+ for cursor-based pagination and field selection.
+ operationId: legacy_observationsV1_getMany
tags:
- - Models
+ - LegacyObservationsV1
parameters:
- name: page
in: query
- description: page number, starts at 1
+ description: Page number, starts at 1.
required: false
schema:
type: integer
nullable: true
- name: limit
in: query
- description: limit of items per page
+ description: >-
+ Limit of items per page. If you encounter api issues due to too
+ large page sizes, try to reduce the limit.
required: false
schema:
type: integer
nullable: true
- responses:
- '200':
- description: ''
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/PaginatedModels'
- '400':
- description: ''
- content:
- application/json:
- schema: {}
- '401':
- description: ''
- content:
- application/json:
- schema: {}
- '403':
- description: ''
- content:
- application/json:
- schema: {}
- '404':
- description: ''
- content:
- application/json:
- schema: {}
- '405':
- description: ''
- content:
- application/json:
- schema: {}
- security: *ref_0
- /api/public/models/{id}:
- put:
- description: >-
- Create or replace a project-owned model using its id. Built-in models
- cannot be modified.
- operationId: models_upsert
- tags:
- - Models
- parameters:
- - name: id
- in: path
- required: true
+ - name: name
+ in: query
+ required: false
schema:
type: string
- responses:
- '200':
- description: ''
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/Model'
- '400':
- description: ''
- content:
- application/json:
- schema: {}
- '401':
- description: ''
- content:
- application/json:
- schema: {}
- '403':
- description: ''
- content:
- application/json:
- schema: {}
- '404':
- description: ''
- content:
- application/json:
- schema: {}
- '405':
- description: ''
- content:
- application/json:
- schema: {}
- security: *ref_0
- requestBody:
- required: true
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/CreateModelRequest'
- get:
- description: Get a model
- operationId: models_get
- tags:
- - Models
- parameters:
- - name: id
- in: path
- required: true
- schema:
- type: string
- responses:
- '200':
- description: ''
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/Model'
- '400':
- description: ''
- content:
- application/json:
- schema: {}
- '401':
- description: ''
- content:
- application/json:
- schema: {}
- '403':
- description: ''
- content:
- application/json:
- schema: {}
- '404':
- description: ''
- content:
- application/json:
- schema: {}
- '405':
- description: ''
- content:
- application/json:
- schema: {}
- security: *ref_0
- delete:
- description: >-
- Delete a model. Cannot delete models managed by Langfuse. You can create
- your own definition with the same modelName to override the definition
- though.
- operationId: models_delete
- tags:
- - Models
- parameters:
- - name: id
- in: path
- required: true
- schema:
- type: string
- responses:
- '204':
- description: ''
- '400':
- description: ''
- content:
- application/json:
- schema: {}
- '401':
- description: ''
- content:
- application/json:
- schema: {}
- '403':
- description: ''
- content:
- application/json:
- schema: {}
- '404':
- description: ''
- content:
- application/json:
- schema: {}
- '405':
- description: ''
- content:
- application/json:
- schema: {}
- security: *ref_0
- /api/public/v2/observations:
- get:
- description: >-
- Get a list of observations with cursor-based pagination and flexible
- field selection.
-
-
- ## Cursor-based Pagination
-
- This endpoint uses cursor-based pagination for efficient traversal of
- large datasets.
-
- The cursor is returned in the response metadata and should be passed in
- subsequent requests
-
- to retrieve the next page of results.
-
-
- ## Field Selection
-
- Use the `fields` parameter to control which observation fields are
- returned:
-
- - `core` - Always included: id, traceId, startTime, endTime, projectId,
- parentObservationId, type
-
- - `basic` - name, level, statusMessage, version, environment,
- bookmarked, public, userId, sessionId, isRootObservation
-
- - `time` - completionStartTime, createdAt, updatedAt
-
- - `io` - input, output
-
- - `metadata` - metadata (truncated to 200 chars by default, use
- `expandMetadata` to get full values)
-
- - `model` - providedModelName, internalModelId, modelParameters
-
- - `usage` - usageDetails, costDetails, totalCost, usagePricingTierName
-
- - `prompt` - promptId, promptName, promptVersion
-
- - `metrics` - latency, timeToFirstToken
-
- - `trace_context` - tags, release, traceName
-
-
- If not specified, `core` and `basic` field groups are returned.
-
-
- ## Filters
-
- Multiple filtering options are available via query parameters or the
- structured `filter` parameter.
-
- When using the `filter` parameter, it takes precedence over individual
- query parameter filters.
- operationId: observations_getMany
- tags:
- - Observations
- parameters:
- - name: fields
- in: query
- description: >-
- Comma-separated list of field groups to include in the response.
-
- Available groups: core, basic, time, io, metadata, model, usage,
- prompt, metrics, trace_context.
-
- If not specified, `core` and `basic` field groups are returned.
-
- Example: "basic,usage,model"
- required: false
- schema:
- type: string
- nullable: true
- - name: expandMetadata
- in: query
- description: |-
- Comma-separated list of metadata keys to return non-truncated.
- By default, metadata values over 200 characters are truncated.
- Use this parameter to retrieve full values for specific keys.
- Example: "key1,key2"
- required: false
- schema:
- type: string
- nullable: true
- - name: limit
- in: query
- description: Number of items to return per page. Maximum 1000, default 50.
- required: false
- schema:
- type: integer
- nullable: true
- - name: cursor
- in: query
- description: >-
- Base64-encoded cursor for pagination. Use the cursor from the
- previous response to get the next page.
- required: false
- schema:
- type: string
- nullable: true
- - name: parseIoAsJson
- in: query
- description: |-
- **Deprecated.** Setting this to `true` will return a 400 error.
- Input/output fields are always returned as raw strings.
- Remove this parameter or set it to `false`.
- required: false
- schema:
- type: boolean
- nullable: true
- - name: name
- in: query
- required: false
- schema:
- type: string
- nullable: true
- - name: userId
- in: query
- required: false
- schema:
- type: string
- nullable: true
- - name: sessionId
- in: query
- description: Filter by session ID.
- required: false
+ nullable: true
+ - name: userId
+ in: query
+ required: false
schema:
type: string
nullable: true
- name: type
in: query
- description: >-
- Filter by observation type (e.g., "GENERATION", "SPAN", "EVENT",
- "AGENT", "TOOL", "CHAIN", "RETRIEVER", "EVALUATOR", "EMBEDDING",
- "GUARDRAIL")
required: false
schema:
type: string
@@ -3587,30 +3117,10 @@ paths:
nullable: true
- name: parentObservationId
in: query
- description: >-
- Filter by the physical parent observation ID.
-
- An empty value matches only observations without a physical parent.
- Use `isRootObservation` to include observations marked as app roots
- by the SDK, which may retain a non-null `parentObservationId`.
required: false
schema:
type: string
nullable: true
- - name: isRootObservation
- in: query
- description: >-
- Filter by whether an observation is a logical root.
-
- Root observations include observations without a physical parent and
- observations marked as app roots by the SDK.
-
- An app-root observation may have `isRootObservation=true` and a
- non-null `parentObservationId`.
- required: false
- schema:
- type: boolean
- nullable: true
- name: environment
in: query
description: >-
@@ -3669,12 +3179,12 @@ paths:
"column": string, // Required. Column to filter on (see available columns below)
"operator": string, // Required. Operator based on type:
// - datetime: ">", "<", ">=", "<="
- // - string: "=", "contains", "does not contain", "starts with", "ends with", "matches"
+ // - string: "=", "contains", "does not contain", "starts with", "ends with"
// - stringOptions: "any of", "none of"
// - categoryOptions: "any of", "none of"
// - arrayOptions: "any of", "none of", "all of"
// - number: "=", ">", "<", ">=", "<="
- // - stringObject: "=", "contains", "does not contain", "starts with", "ends with", "matches"
+ // - stringObject: "=", "contains", "does not contain", "starts with", "ends with"
// - numberObject: "=", ">", "<", ">=", "<="
// - boolean: "=", "<>"
// - null: "is null", "is not null"
@@ -3711,23 +3221,6 @@ paths:
- `version` (string) - Version tag
- - `userId` (string) - User ID
-
- - `sessionId` (string) - Session ID
-
- - `isRootObservation` (boolean) - Whether the observation is a
- logical root. Observations marked as app roots by the SDK may retain
- a non-null parentObservationId.
-
-
- ### Trace-Related Fields
-
- - `traceName` (string) - Name of the parent trace
-
- - `traceTags` (arrayOptions) - Tags from the parent trace
-
- - `tags` (arrayOptions) - Alias for traceTags
-
### Performance Metrics
@@ -3759,8 +3252,7 @@ paths:
### Model Information
- - `model` (string) - Provided model name (alias:
- `providedModelName`)
+ - `model` (string) - Provided model name
- `promptName` (string) - Associated prompt name
@@ -3769,28 +3261,20 @@ paths:
### Structured Data
- - `input` (string) - Observation input. Supports accelerated indexed
- literal search with the `matches` operator.
-
- - `output` (string) - Observation output. Supports accelerated
- indexed literal search with the `matches` operator.
-
- `metadata` (stringObject/numberObject/categoryOptions) - Metadata
key-value pairs. Use `key` parameter to filter on specific metadata
keys.
- The `matches` operator is only supported for `input`, `output`, and
- stringObject `metadata` filters. It performs indexed literal search
- with token-boundary pruning using the events table text indexes.
- Case sensitivity differs by target: `input` and `output` matches are
- case-insensitive, while metadata value matches are case-sensitive.
- Unlike SQL `LIKE`, `%` and `_` are treated as literal characters.
- Use `contains` for legacy substring semantics where the API allows
- it. Any v2 `input` or `output` filter must be accompanied by at
- least one `=` or `matches` filter on `input` or `output`; standalone
- `contains`, `starts with`, `ends with`, and `does not contain`
- filters on these columns are rejected.
+ ### Associated Trace Fields (requires join with traces table)
+
+ - `userId` (string) - User ID from associated trace
+
+ - `traceName` (string) - Name from associated trace
+
+ - `traceEnvironment` (string) - Environment from associated trace
+
+ - `traceTags` (arrayOptions) - Tags from associated trace
## Filter Examples
@@ -3816,18 +3300,6 @@ paths:
"key": "environment",
"operator": "=",
"value": "production"
- },
- {
- "type": "string",
- "column": "output",
- "operator": "matches",
- "value": "needle"
- },
- {
- "type": "boolean",
- "column": "isRootObservation",
- "operator": "=",
- "value": true
}
]
@@ -3842,7 +3314,7 @@ paths:
content:
application/json:
schema:
- $ref: '#/components/schemas/ObservationsV2Response'
+ $ref: '#/components/schemas/legacyObservationsViews'
'400':
description: ''
content:
@@ -3869,51 +3341,23 @@ paths:
application/json:
schema: {}
security: *ref_0
- /api/public/otel/v1/traces:
- post:
- description: >-
- **OpenTelemetry Traces Ingestion Endpoint**
-
-
- This endpoint implements the OTLP/HTTP specification for trace
- ingestion, providing native OpenTelemetry integration for Langfuse
- Observability.
-
-
- **Supported Formats:**
-
- - Binary Protobuf: `Content-Type: application/x-protobuf`
-
- - JSON Protobuf: `Content-Type: application/json`
-
- - Supports gzip compression via `Content-Encoding: gzip` header
-
-
- **Specification Compliance:**
-
- - Conforms to [OTLP/HTTP Trace
- Export](https://opentelemetry.io/docs/specs/otlp/#otlphttp)
-
- - Implements `ExportTraceServiceRequest` message format
-
-
- **Documentation:**
-
- - Integration guide:
- https://langfuse.com/integrations/native/opentelemetry
-
- - Data model: https://langfuse.com/docs/observability/data-model
- operationId: opentelemetry_exportTraces
+ deprecated: true
+ /api/public/scores/{scoreId}:
+ delete:
+ description: Delete a score (supports both trace and session scores)
+ operationId: legacy_scoreV1_delete
tags:
- - Opentelemetry
- parameters: []
+ - LegacyScoreV1
+ parameters:
+ - name: scoreId
+ in: path
+ description: The unique langfuse identifier of a score
+ required: true
+ schema:
+ type: string
responses:
- '200':
+ '204':
description: ''
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/OtelTraceResponse'
'400':
description: ''
content:
@@ -3940,38 +3384,34 @@ paths:
application/json:
schema: {}
security: *ref_0
- requestBody:
- required: true
- content:
- application/json:
- schema:
- type: object
- properties:
- resourceSpans:
- type: array
- items:
- $ref: '#/components/schemas/OtelResourceSpan'
- description: >-
- Array of resource spans containing trace data as defined in
- the OTLP specification
- required:
- - resourceSpans
- /api/public/organizations/memberships:
+ /api/public/llm-connections:
get:
- description: >-
- Get all memberships for the organization associated with the API key
- (requires organization-scoped API key)
- operationId: organizations_getOrganizationMemberships
+ description: Get all LLM connections in a project
+ operationId: llmConnections_list
tags:
- - Organizations
- parameters: []
+ - LlmConnections
+ parameters:
+ - name: page
+ in: query
+ description: page number, starts at 1
+ required: false
+ schema:
+ type: integer
+ nullable: true
+ - name: limit
+ in: query
+ description: limit of items per page
+ required: false
+ schema:
+ type: integer
+ nullable: true
responses:
'200':
description: ''
content:
application/json:
schema:
- $ref: '#/components/schemas/MembershipsResponse'
+ $ref: '#/components/schemas/PaginatedLlmConnections'
'400':
description: ''
content:
@@ -4000,11 +3440,11 @@ paths:
security: *ref_0
put:
description: >-
- Create or update a membership for the organization associated with the
- API key (requires organization-scoped API key)
- operationId: organizations_updateOrganizationMembership
+ Create or update an LLM connection. The connection is upserted on
+ provider.
+ operationId: llmConnections_upsert
tags:
- - Organizations
+ - LlmConnections
parameters: []
responses:
'200':
@@ -4012,7 +3452,7 @@ paths:
content:
application/json:
schema:
- $ref: '#/components/schemas/MembershipResponse'
+ $ref: '#/components/schemas/LlmConnection'
'400':
description: ''
content:
@@ -4044,64 +3484,17 @@ paths:
content:
application/json:
schema:
- $ref: '#/components/schemas/MembershipRequest'
+ $ref: '#/components/schemas/UpsertLlmConnectionRequest'
+ /api/public/llm-connections/{id}:
delete:
description: >-
- Delete a membership from the organization associated with the API key
- (requires organization-scoped API key)
- operationId: organizations_deleteOrganizationMembership
- tags:
- - Organizations
- parameters: []
- responses:
- '200':
- description: ''
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/MembershipDeletionResponse'
- '400':
- description: ''
- content:
- application/json:
- schema: {}
- '401':
- description: ''
- content:
- application/json:
- schema: {}
- '403':
- description: ''
- content:
- application/json:
- schema: {}
- '404':
- description: ''
- content:
- application/json:
- schema: {}
- '405':
- description: ''
- content:
- application/json:
- schema: {}
- security: *ref_0
- requestBody:
- required: true
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/DeleteMembershipRequest'
- /api/public/projects/{projectId}/memberships:
- get:
- description: >-
- Get all memberships for a specific project (requires organization-scoped
- API key)
- operationId: organizations_getProjectMemberships
+ Delete an LLM connection by id. Evaluators that depend on the deleted
+ connection are automatically paused.
+ operationId: llmConnections_delete
tags:
- - Organizations
+ - LlmConnections
parameters:
- - name: projectId
+ - name: id
in: path
required: true
schema:
@@ -4112,7 +3505,7 @@ paths:
content:
application/json:
schema:
- $ref: '#/components/schemas/MembershipsResponse'
+ $ref: '#/components/schemas/DeleteLlmConnectionResponse'
'400':
description: ''
content:
@@ -4139,17 +3532,16 @@ paths:
application/json:
schema: {}
security: *ref_0
- put:
- description: >-
- Create or update a membership for a specific project (requires
- organization-scoped API key). The user must already be a member of the
- organization.
- operationId: organizations_updateProjectMembership
+ /api/public/media/{mediaId}:
+ get:
+ description: Get a media record
+ operationId: media_get
tags:
- - Organizations
+ - Media
parameters:
- - name: projectId
+ - name: mediaId
in: path
+ description: The unique langfuse identifier of a media record
required: true
schema:
type: string
@@ -4159,7 +3551,7 @@ paths:
content:
application/json:
schema:
- $ref: '#/components/schemas/MembershipResponse'
+ $ref: '#/components/schemas/GetMediaResponse'
'400':
description: ''
content:
@@ -4186,33 +3578,21 @@ paths:
application/json:
schema: {}
security: *ref_0
- requestBody:
- required: true
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/MembershipRequest'
- delete:
- description: >-
- Delete a membership from a specific project (requires
- organization-scoped API key). The user must be a member of the
- organization.
- operationId: organizations_deleteProjectMembership
+ patch:
+ description: Patch a media record
+ operationId: media_patch
tags:
- - Organizations
+ - Media
parameters:
- - name: projectId
+ - name: mediaId
in: path
+ description: The unique langfuse identifier of a media record
required: true
schema:
type: string
responses:
- '200':
+ '204':
description: ''
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/MembershipDeletionResponse'
'400':
description: ''
content:
@@ -4244,15 +3624,13 @@ paths:
content:
application/json:
schema:
- $ref: '#/components/schemas/DeleteMembershipRequest'
- /api/public/organizations/projects:
- get:
- description: >-
- Get all projects for the organization associated with the API key
- (requires organization-scoped API key)
- operationId: organizations_getOrganizationProjects
+ $ref: '#/components/schemas/PatchMediaBody'
+ /api/public/media:
+ post:
+ description: Get a presigned upload URL for a media record
+ operationId: media_getUploadUrl
tags:
- - Organizations
+ - Media
parameters: []
responses:
'200':
@@ -4260,7 +3638,7 @@ paths:
content:
application/json:
schema:
- $ref: '#/components/schemas/OrganizationProjectsResponse'
+ $ref: '#/components/schemas/GetMediaUploadUrlResponse'
'400':
description: ''
content:
@@ -4287,24 +3665,1467 @@ paths:
application/json:
schema: {}
security: *ref_0
- /api/public/organizations/apiKeys:
+ requestBody:
+ required: true
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/GetMediaUploadUrlRequest'
+ /api/public/v2/metrics:
get:
description: >-
- Get all API keys for the organization associated with the API key
- (requires organization-scoped API key)
- operationId: organizations_getOrganizationApiKeys
- tags:
- - Organizations
- parameters: []
- responses:
- '200':
- description: ''
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/OrganizationApiKeysResponse'
- '400':
- description: ''
+ Get metrics from the Langfuse project using a query object. V2 endpoint
+ with optimized performance.
+
+
+ ## V2 Differences
+
+ - Supports `observations`, `scores-numeric`, `scores-boolean`, and
+ `scores-categorical` views only (traces view not supported)
+
+ - Direct access to tags and release fields on observations
+
+ - Semantic-root filtering and grouping through the v2-only
+ `isRootObservation` dimension
+
+ - Backwards-compatible: traceName, traceRelease, traceVersion dimensions
+ are still available on observations view
+
+ - High cardinality dimensions are not supported and will return a 400
+ error (see below)
+
+
+ For more details, see the [Metrics API
+ documentation](https://langfuse.com/docs/metrics/features/metrics-api).
+
+
+ ## Available Views
+
+
+ ### observations
+
+ Query observation-level data (spans, generations, events).
+
+
+ **Dimensions:**
+
+ - `environment` - Deployment environment (e.g., production, staging)
+
+ - `type` - Type of observation (SPAN, GENERATION, EVENT)
+
+ - `name` - Name of the observation
+
+ - `level` - Logging level of the observation
+
+ - `version` - Version of the observation
+
+ - `tags` - User-defined tags
+
+ - `release` - Release version
+
+ - `traceName` - Name of the parent trace (backwards-compatible)
+
+ - `traceRelease` - Release version of the parent trace
+ (backwards-compatible, maps to release)
+
+ - `traceVersion` - Version of the parent trace (backwards-compatible,
+ maps to version)
+
+ - `providedModelName` - Name of the model used
+
+ - `promptName` - Name of the prompt used
+
+ - `promptVersion` - Version of the prompt used
+
+ - `isRootObservation` - Boolean semantic-root status. `true` includes
+ physical roots and app roots whose SDK parent is external (so
+ `parentObservationId` may be non-null).
+
+ - `startTimeMonth` - Month of start_time in YYYY-MM format
+
+
+ **Measures:**
+
+ - `count` - Total number of observations
+
+ - `latency` - Observation latency (milliseconds)
+
+ - `streamingLatency` - Generation latency from completion start to end
+ (milliseconds)
+
+ - `inputTokens` - Sum of input tokens consumed
+
+ - `outputTokens` - Sum of output tokens produced
+
+ - `totalTokens` - Sum of all tokens consumed
+
+ - `outputTokensPerSecond` - Output tokens per second
+
+ - `tokensPerSecond` - Total tokens per second
+
+ - `inputCost` - Input cost (USD)
+
+ - `outputCost` - Output cost (USD)
+
+ - `totalCost` - Total cost (USD)
+
+ - `timeToFirstToken` - Time to first token (milliseconds)
+
+ - `countScores` - Number of scores attached to the observation
+
+
+ ### scores-numeric
+
+ Query numeric and boolean score data.
+
+
+ **Dimensions:**
+
+ - `environment` - Deployment environment
+
+ - `name` - Name of the score (e.g., accuracy, toxicity)
+
+ - `source` - Origin of the score (API, ANNOTATION, EVAL)
+
+ - `dataType` - Data type (NUMERIC, BOOLEAN)
+
+ - `configId` - Identifier of the score config
+
+ - `timestampMonth` - Month in YYYY-MM format
+
+ - `timestampDay` - Day in YYYY-MM-DD format
+
+ - `value` - Numeric value of the score
+
+ - `traceName` - Name of the parent trace
+
+ - `tags` - Tags
+
+ - `traceRelease` - Release version
+
+ - `traceVersion` - Version
+
+ - `observationName` - Name of the associated observation
+
+ - `observationModelName` - Model name of the associated observation
+
+ - `observationPromptName` - Prompt name of the associated observation
+
+ - `observationPromptVersion` - Prompt version of the associated
+ observation
+
+
+ **Measures:**
+
+ - `count` - Total number of scores
+
+ - `value` - Score value (for aggregations)
+
+
+ ### scores-boolean
+
+ Query boolean score data. It has the same score and parent
+ trace/observation dimensions as scores-numeric, plus:
+
+
+ **Dimensions:**
+
+ - `booleanValue` - Boolean value for true/false grouping and filtering
+
+
+ **Measures:**
+
+ - `count` - Total number of boolean scores
+
+ - `value` - Numeric 0/1 score value; `avg` returns the true-rate
+
+
+ ### scores-categorical
+
+ Query categorical score data. Same dimensions as scores-numeric except
+ uses `stringValue` instead of `value`.
+
+
+ **Measures:**
+
+ - `count` - Total number of scores
+
+
+ ## High Cardinality Dimensions
+
+ The following dimensions cannot be used as grouping dimensions in v2
+ metrics API as they can cause performance issues.
+
+ Use them in filters instead.
+
+
+ **observations view:**
+
+ - `id` - Use traceId filter to narrow down results
+
+ - `traceId` - Use traceId filter instead
+
+ - `userId` - Use userId filter instead
+
+ - `sessionId` - Use sessionId filter instead
+
+ - `parentObservationId` - Use parentObservationId filter instead
+
+
+ **scores-numeric / scores-boolean / scores-categorical views:**
+
+ - `id` - Use specific filters to narrow down results
+
+ - `traceId` - Use traceId filter instead
+
+ - `userId` - Use userId filter instead
+
+ - `sessionId` - Use sessionId filter instead
+
+ - `observationId` - Use observationId filter instead
+
+
+ ## Aggregations
+
+ Available aggregation functions: `sum`, `avg`, `count`, `max`, `min`,
+ `p50`, `p75`, `p90`, `p95`, `p99`, `histogram`
+
+
+ ## Time Granularities
+
+ Available granularities for timeDimension: `auto`, `minute`, `hour`,
+ `day`, `week`, `month`
+
+ - `auto` bins the data into approximately 50 buckets based on the time
+ range
+ operationId: metrics_metrics
+ tags:
+ - Metrics
+ parameters:
+ - name: query
+ in: query
+ description: >-
+ JSON string containing the query parameters with the following
+ structure:
+
+ ```json
+
+ {
+ "view": string, // Required. One of "observations", "scores-numeric", "scores-boolean", "scores-categorical"
+ "dimensions": [ // Optional. Default: []
+ {
+ "field": string // Field to group by (see available dimensions above)
+ }
+ ],
+ "metrics": [ // Required. At least one metric must be provided
+ {
+ "measure": string, // What to measure (see available measures above)
+ "aggregation": string // How to aggregate: "sum", "avg", "count", "max", "min", "p50", "p75", "p90", "p95", "p99", "histogram"
+ }
+ ],
+ "filters": [ // Optional. Default: []
+ {
+ "column": string, // Column to filter on (any dimension field)
+ "operator": string, // Operator based on type:
+ // - datetime: ">", "<", ">=", "<="
+ // - string: "=", "contains", "does not contain", "starts with", "ends with"
+ // - stringOptions: "any of", "none of"
+ // - arrayOptions: "any of", "none of", "all of"
+ // - number: "=", ">", "<", ">=", "<="
+ // - stringObject/numberObject: same as string/number with required "key"
+ // - boolean: "=", "<>"
+ // - null: "is null", "is not null"
+ "value": any, // Value to compare against
+ "type": string, // Data type: "datetime", "string", "number", "stringOptions", "categoryOptions", "arrayOptions", "stringObject", "numberObject", "boolean", "null"
+ "key": string // Required only for stringObject/numberObject types (e.g., metadata filtering)
+ }
+ ],
+ "timeDimension": { // Optional. Default: null. If provided, results will be grouped by time
+ "granularity": string // One of "auto", "minute", "hour", "day", "week", "month"
+ },
+ "fromTimestamp": string, // Required. ISO datetime string for start of time range
+ "toTimestamp": string, // Required. ISO datetime string for end of time range (must be after fromTimestamp)
+ "orderBy": [ // Optional. Default: null
+ {
+ "field": string, // Field to order by (dimension or metric alias)
+ "direction": string // "asc" or "desc"
+ }
+ ],
+ "config": { // Optional. Query-specific configuration
+ "bins": number, // Optional. Number of bins for histogram aggregation (1-100), default: 10
+ "row_limit": number // Optional. Maximum number of rows to return (1-1000), default: 100
+ }
+ }
+
+ ```
+
+
+ For example, to count semantic roots (including app roots with a
+ non-null external parent), use a boolean filter:
+
+ ```json
+
+ {
+ "view": "observations",
+ "metrics": [{"measure": "count", "aggregation": "count"}],
+ "filters": [{"column": "isRootObservation", "operator": "=", "value": true, "type": "boolean"}],
+ "fromTimestamp": "2025-01-01T00:00:00.000Z",
+ "toTimestamp": "2025-02-01T00:00:00.000Z"
+ }
+
+ ```
+ required: true
+ schema:
+ type: string
+ responses:
+ '200':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/MetricsV2Response'
+ '400':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '401':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '403':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '404':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '405':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ security: *ref_0
+ /api/public/models:
+ post:
+ description: Create a model
+ operationId: models_create
+ tags:
+ - Models
+ parameters: []
+ responses:
+ '200':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/Model'
+ '400':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '401':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '403':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '404':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '405':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ security: *ref_0
+ requestBody:
+ required: true
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/CreateModelRequest'
+ get:
+ description: Get all models
+ operationId: models_list
+ tags:
+ - Models
+ parameters:
+ - name: page
+ in: query
+ description: page number, starts at 1
+ required: false
+ schema:
+ type: integer
+ nullable: true
+ - name: limit
+ in: query
+ description: limit of items per page
+ required: false
+ schema:
+ type: integer
+ nullable: true
+ responses:
+ '200':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/PaginatedModels'
+ '400':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '401':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '403':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '404':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '405':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ security: *ref_0
+ /api/public/models/{id}:
+ put:
+ description: >-
+ Create or replace a project-owned model using its id. Built-in models
+ cannot be modified.
+ operationId: models_upsert
+ tags:
+ - Models
+ parameters:
+ - name: id
+ in: path
+ required: true
+ schema:
+ type: string
+ responses:
+ '200':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/Model'
+ '400':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '401':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '403':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '404':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '405':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ security: *ref_0
+ requestBody:
+ required: true
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/CreateModelRequest'
+ get:
+ description: Get a model
+ operationId: models_get
+ tags:
+ - Models
+ parameters:
+ - name: id
+ in: path
+ required: true
+ schema:
+ type: string
+ responses:
+ '200':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/Model'
+ '400':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '401':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '403':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '404':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '405':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ security: *ref_0
+ delete:
+ description: >-
+ Delete a model. Cannot delete models managed by Langfuse. You can create
+ your own definition with the same modelName to override the definition
+ though.
+ operationId: models_delete
+ tags:
+ - Models
+ parameters:
+ - name: id
+ in: path
+ required: true
+ schema:
+ type: string
+ responses:
+ '204':
+ description: ''
+ '400':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '401':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '403':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '404':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '405':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ security: *ref_0
+ /api/public/v2/observations:
+ get:
+ description: >-
+ Get a list of observations with cursor-based pagination and flexible
+ field selection.
+
+
+ ## Cursor-based Pagination
+
+ This endpoint uses cursor-based pagination for efficient traversal of
+ large datasets.
+
+ The cursor is returned in the response metadata and should be passed in
+ subsequent requests
+
+ to retrieve the next page of results.
+
+
+ ## Field Selection
+
+ Use the `fields` parameter to control which observation fields are
+ returned:
+
+ - `core` - Always included: id, traceId, startTime, endTime, projectId,
+ parentObservationId, type
+
+ - `basic` - name, level, statusMessage, version, environment,
+ bookmarked, public, userId, sessionId, isRootObservation
+
+ - `time` - completionStartTime, createdAt, updatedAt
+
+ - `io` - input, output
+
+ - `metadata` - metadata (truncated to 200 chars by default, use
+ `expandMetadata` to get full values)
+
+ - `model` - providedModelName, internalModelId, modelParameters
+
+ - `usage` - usageDetails, costDetails, totalCost, usagePricingTierName
+
+ - `prompt` - promptId, promptName, promptVersion
+
+ - `metrics` - latency, timeToFirstToken
+
+ - `trace_context` - tags, release, traceName
+
+
+ If not specified, `core` and `basic` field groups are returned.
+
+
+ ## Filters
+
+ Multiple filtering options are available via query parameters or the
+ structured `filter` parameter.
+
+ When using the `filter` parameter, it takes precedence over individual
+ query parameter filters.
+ operationId: observations_getMany
+ tags:
+ - Observations
+ parameters:
+ - name: fields
+ in: query
+ description: >-
+ Comma-separated list of field groups to include in the response.
+
+ Available groups: core, basic, time, io, metadata, model, usage,
+ prompt, metrics, trace_context.
+
+ If not specified, `core` and `basic` field groups are returned.
+
+ Example: "basic,usage,model"
+ required: false
+ schema:
+ type: string
+ nullable: true
+ - name: expandMetadata
+ in: query
+ description: |-
+ Comma-separated list of metadata keys to return non-truncated.
+ By default, metadata values over 200 characters are truncated.
+ Use this parameter to retrieve full values for specific keys.
+ Example: "key1,key2"
+ required: false
+ schema:
+ type: string
+ nullable: true
+ - name: limit
+ in: query
+ description: Number of items to return per page. Maximum 1000, default 50.
+ required: false
+ schema:
+ type: integer
+ nullable: true
+ - name: cursor
+ in: query
+ description: >-
+ Base64-encoded cursor for pagination. Use the cursor from the
+ previous response to get the next page.
+ required: false
+ schema:
+ type: string
+ nullable: true
+ - name: parseIoAsJson
+ in: query
+ description: |-
+ **Deprecated.** Setting this to `true` will return a 400 error.
+ Input/output fields are always returned as raw strings.
+ Remove this parameter or set it to `false`.
+ required: false
+ schema:
+ type: boolean
+ nullable: true
+ - name: name
+ in: query
+ required: false
+ schema:
+ type: string
+ nullable: true
+ - name: userId
+ in: query
+ required: false
+ schema:
+ type: string
+ nullable: true
+ - name: sessionId
+ in: query
+ description: Filter by session ID.
+ required: false
+ schema:
+ type: string
+ nullable: true
+ - name: type
+ in: query
+ description: >-
+ Filter by observation type (e.g., "GENERATION", "SPAN", "EVENT",
+ "AGENT", "TOOL", "CHAIN", "RETRIEVER", "EVALUATOR", "EMBEDDING",
+ "GUARDRAIL")
+ required: false
+ schema:
+ type: string
+ nullable: true
+ - name: traceId
+ in: query
+ required: false
+ schema:
+ type: string
+ nullable: true
+ - name: level
+ in: query
+ description: >-
+ Optional filter for observations with a specific level (e.g.
+ "DEBUG", "DEFAULT", "WARNING", "ERROR").
+ required: false
+ schema:
+ $ref: '#/components/schemas/ObservationLevel'
+ nullable: true
+ - name: parentObservationId
+ in: query
+ description: >-
+ Filter by the physical parent observation ID.
+
+ An empty value matches only observations without a physical parent.
+ Use `isRootObservation` to include observations marked as app roots
+ by the SDK, which may retain a non-null `parentObservationId`.
+ required: false
+ schema:
+ type: string
+ nullable: true
+ - name: isRootObservation
+ in: query
+ description: >-
+ Filter by whether an observation is a logical root.
+
+ Root observations include observations without a physical parent and
+ observations marked as app roots by the SDK.
+
+ An app-root observation may have `isRootObservation=true` and a
+ non-null `parentObservationId`.
+ required: false
+ schema:
+ type: boolean
+ nullable: true
+ - name: environment
+ in: query
+ description: >-
+ Optional filter for observations where the environment is one of the
+ provided values.
+ required: false
+ schema:
+ type: array
+ items:
+ type: string
+ nullable: true
+ - name: fromStartTime
+ in: query
+ description: >-
+ Retrieve only observations with a start_time on or after this
+ datetime (ISO 8601).
+ required: false
+ schema:
+ type: string
+ format: date-time
+ nullable: true
+ - name: toStartTime
+ in: query
+ description: >-
+ Retrieve only observations with a start_time before this datetime
+ (ISO 8601).
+ required: false
+ schema:
+ type: string
+ format: date-time
+ nullable: true
+ - name: version
+ in: query
+ description: Optional filter to only include observations with a certain version.
+ required: false
+ schema:
+ type: string
+ nullable: true
+ - name: filter
+ in: query
+ description: >-
+ JSON string containing an array of filter conditions. When provided,
+ this takes precedence over query parameter filters (userId, name,
+ type, level, environment, fromStartTime, ...).
+
+
+ ## Filter Structure
+
+ Each filter condition has the following structure:
+
+ ```json
+
+ [
+ {
+ "type": string, // Required. One of: "datetime", "string", "number", "stringOptions", "categoryOptions", "arrayOptions", "stringObject", "numberObject", "boolean", "null"
+ "column": string, // Required. Column to filter on (see available columns below)
+ "operator": string, // Required. Operator based on type:
+ // - datetime: ">", "<", ">=", "<="
+ // - string: "=", "contains", "does not contain", "starts with", "ends with", "matches"
+ // - stringOptions: "any of", "none of"
+ // - categoryOptions: "any of", "none of"
+ // - arrayOptions: "any of", "none of", "all of"
+ // - number: "=", ">", "<", ">=", "<="
+ // - stringObject: "=", "contains", "does not contain", "starts with", "ends with", "matches"
+ // - numberObject: "=", ">", "<", ">=", "<="
+ // - boolean: "=", "<>"
+ // - null: "is null", "is not null"
+ "value": any, // Required (except for null type). Value to compare against. Type depends on filter type
+ "key": string // Required only for stringObject, numberObject, and categoryOptions types when filtering on nested fields like metadata
+ }
+ ]
+
+ ```
+
+
+ ## Available Columns
+
+
+ ### Core Observation Fields
+
+ - `id` (string) - Observation ID
+
+ - `type` (string) - Observation type (SPAN, GENERATION, EVENT)
+
+ - `name` (string) - Observation name
+
+ - `traceId` (string) - Associated trace ID
+
+ - `startTime` (datetime) - Observation start time
+
+ - `endTime` (datetime) - Observation end time
+
+ - `environment` (string) - Environment tag
+
+ - `level` (string) - Log level (DEBUG, DEFAULT, WARNING, ERROR)
+
+ - `statusMessage` (string) - Status message
+
+ - `version` (string) - Version tag
+
+ - `userId` (string) - User ID
+
+ - `sessionId` (string) - Session ID
+
+ - `isRootObservation` (boolean) - Whether the observation is a
+ logical root. Observations marked as app roots by the SDK may retain
+ a non-null parentObservationId.
+
+
+ ### Trace-Related Fields
+
+ - `traceName` (string) - Name of the parent trace
+
+ - `traceTags` (arrayOptions) - Tags from the parent trace
+
+ - `tags` (arrayOptions) - Alias for traceTags
+
+
+ ### Performance Metrics
+
+ - `latency` (number) - Latency in seconds (calculated: end_time -
+ start_time)
+
+ - `timeToFirstToken` (number) - Time to first token in seconds
+
+ - `tokensPerSecond` (number) - Output tokens per second
+
+
+ ### Token Usage
+
+ - `inputTokens` (number) - Number of input tokens
+
+ - `outputTokens` (number) - Number of output tokens
+
+ - `totalTokens` (number) - Total tokens (alias: `tokens`)
+
+
+ ### Cost Metrics
+
+ - `inputCost` (number) - Input cost in USD
+
+ - `outputCost` (number) - Output cost in USD
+
+ - `totalCost` (number) - Total cost in USD
+
+
+ ### Model Information
+
+ - `model` (string) - Provided model name (alias:
+ `providedModelName`)
+
+ - `promptName` (string) - Associated prompt name
+
+ - `promptVersion` (number) - Associated prompt version
+
+
+ ### Structured Data
+
+ - `input` (string) - Observation input. Supports accelerated indexed
+ literal search with the `matches` operator.
+
+ - `output` (string) - Observation output. Supports accelerated
+ indexed literal search with the `matches` operator.
+
+ - `metadata` (stringObject/numberObject/categoryOptions) - Metadata
+ key-value pairs. Use `key` parameter to filter on specific metadata
+ keys.
+
+
+ The `matches` operator is only supported for `input`, `output`, and
+ stringObject `metadata` filters. It performs indexed literal search
+ with token-boundary pruning using the events table text indexes.
+ Case sensitivity differs by target: `input` and `output` matches are
+ case-insensitive, while metadata value matches are case-sensitive.
+ Unlike SQL `LIKE`, `%` and `_` are treated as literal characters.
+ Use `contains` for legacy substring semantics where the API allows
+ it. Any v2 `input` or `output` filter must be accompanied by at
+ least one `=` or `matches` filter on `input` or `output`; standalone
+ `contains`, `starts with`, `ends with`, and `does not contain`
+ filters on these columns are rejected.
+
+
+ ## Filter Examples
+
+ ```json
+
+ [
+ {
+ "type": "string",
+ "column": "type",
+ "operator": "=",
+ "value": "GENERATION"
+ },
+ {
+ "type": "number",
+ "column": "latency",
+ "operator": ">=",
+ "value": 2.5
+ },
+ {
+ "type": "stringObject",
+ "column": "metadata",
+ "key": "environment",
+ "operator": "=",
+ "value": "production"
+ },
+ {
+ "type": "string",
+ "column": "output",
+ "operator": "matches",
+ "value": "needle"
+ },
+ {
+ "type": "boolean",
+ "column": "isRootObservation",
+ "operator": "=",
+ "value": true
+ }
+ ]
+
+ ```
+ required: false
+ schema:
+ type: string
+ nullable: true
+ responses:
+ '200':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/ObservationsV2Response'
+ '400':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '401':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '403':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '404':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '405':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ security: *ref_0
+ /api/public/otel/v1/traces:
+ post:
+ description: >-
+ **OpenTelemetry Traces Ingestion Endpoint**
+
+
+ This endpoint implements the OTLP/HTTP specification for trace
+ ingestion, providing native OpenTelemetry integration for Langfuse
+ Observability.
+
+
+ **Supported Formats:**
+
+ - Binary Protobuf: `Content-Type: application/x-protobuf`
+
+ - JSON Protobuf: `Content-Type: application/json`
+
+ - Supports gzip compression via `Content-Encoding: gzip` header
+
+
+ **Specification Compliance:**
+
+ - Conforms to [OTLP/HTTP Trace
+ Export](https://opentelemetry.io/docs/specs/otlp/#otlphttp)
+
+ - Implements `ExportTraceServiceRequest` message format
+
+
+ **Documentation:**
+
+ - Integration guide:
+ https://langfuse.com/integrations/native/opentelemetry
+
+ - Data model: https://langfuse.com/docs/observability/data-model
+ operationId: opentelemetry_exportTraces
+ tags:
+ - Opentelemetry
+ parameters: []
+ responses:
+ '200':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/OtelTraceResponse'
+ '400':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '401':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '403':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '404':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '405':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ security: *ref_0
+ requestBody:
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ properties:
+ resourceSpans:
+ type: array
+ items:
+ $ref: '#/components/schemas/OtelResourceSpan'
+ description: >-
+ Array of resource spans containing trace data as defined in
+ the OTLP specification
+ required:
+ - resourceSpans
+ /api/public/organizations/memberships:
+ get:
+ description: >-
+ Get all memberships for the organization associated with the API key
+ (requires organization-scoped API key)
+ operationId: organizations_getOrganizationMemberships
+ tags:
+ - Organizations
+ parameters: []
+ responses:
+ '200':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/MembershipsResponse'
+ '400':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '401':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '403':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '404':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '405':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ security: *ref_0
+ put:
+ description: >-
+ Create or update a membership for the organization associated with the
+ API key (requires organization-scoped API key)
+ operationId: organizations_updateOrganizationMembership
+ tags:
+ - Organizations
+ parameters: []
+ responses:
+ '200':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/MembershipResponse'
+ '400':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '401':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '403':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '404':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '405':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ security: *ref_0
+ requestBody:
+ required: true
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/MembershipRequest'
+ delete:
+ description: >-
+ Delete a membership from the organization associated with the API key
+ (requires organization-scoped API key)
+ operationId: organizations_deleteOrganizationMembership
+ tags:
+ - Organizations
+ parameters: []
+ responses:
+ '200':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/MembershipDeletionResponse'
+ '400':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '401':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '403':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '404':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '405':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ security: *ref_0
+ requestBody:
+ required: true
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/DeleteMembershipRequest'
+ /api/public/projects/{projectId}/memberships:
+ get:
+ description: >-
+ Get all memberships for a specific project (requires organization-scoped
+ API key)
+ operationId: organizations_getProjectMemberships
+ tags:
+ - Organizations
+ parameters:
+ - name: projectId
+ in: path
+ required: true
+ schema:
+ type: string
+ responses:
+ '200':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/MembershipsResponse'
+ '400':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '401':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '403':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '404':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '405':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ security: *ref_0
+ put:
+ description: >-
+ Create or update a membership for a specific project (requires
+ organization-scoped API key). The user must already be a member of the
+ organization.
+ operationId: organizations_updateProjectMembership
+ tags:
+ - Organizations
+ parameters:
+ - name: projectId
+ in: path
+ required: true
+ schema:
+ type: string
+ responses:
+ '200':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/MembershipResponse'
+ '400':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '401':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '403':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '404':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '405':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ security: *ref_0
+ requestBody:
+ required: true
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/MembershipRequest'
+ delete:
+ description: >-
+ Delete a membership from a specific project (requires
+ organization-scoped API key). The user must be a member of the
+ organization.
+ operationId: organizations_deleteProjectMembership
+ tags:
+ - Organizations
+ parameters:
+ - name: projectId
+ in: path
+ required: true
+ schema:
+ type: string
+ responses:
+ '200':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/MembershipDeletionResponse'
+ '400':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '401':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '403':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '404':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '405':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ security: *ref_0
+ requestBody:
+ required: true
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/DeleteMembershipRequest'
+ /api/public/organizations/projects:
+ get:
+ description: >-
+ Get all projects for the organization associated with the API key
+ (requires organization-scoped API key)
+ operationId: organizations_getOrganizationProjects
+ tags:
+ - Organizations
+ parameters: []
+ responses:
+ '200':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/OrganizationProjectsResponse'
+ '400':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '401':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '403':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '404':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '405':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ security: *ref_0
+ /api/public/organizations/apiKeys:
+ get:
+ description: >-
+ Get all API keys for the organization associated with the API key
+ (requires organization-scoped API key)
+ operationId: organizations_getOrganizationApiKeys
+ tags:
+ - Organizations
+ parameters: []
+ responses:
+ '200':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/OrganizationApiKeysResponse'
+ '400':
+ description: ''
content:
application/json:
schema: {}
@@ -6714,83 +7535,547 @@ paths:
These fields aggregate observation levels within the trace:
- - `level` (string) - Highest severity level (ERROR > WARNING >
- DEFAULT > DEBUG)
+ - `level` (string) - Highest severity level (ERROR > WARNING >
+ DEFAULT > DEBUG)
+
+ - `warningCount` (number) - Count of WARNING level observations
+
+ - `errorCount` (number) - Count of ERROR level observations
+
+ - `defaultCount` (number) - Count of DEFAULT level observations
+
+ - `debugCount` (number) - Count of DEBUG level observations
+
+
+ ### Scores (requires join with scores table)
+
+ - `scores_avg` (number) - Average of numeric scores (alias:
+ `scores`)
+
+ - `score_categories` (categoryOptions) - Categorical score values
+
+ - `score_booleans` (booleanObject) - Boolean score values. Use `key`
+ for the score name and a boolean `value`, e.g. `{"type":
+ "booleanObject", "column": "score_booleans", "key": "is_correct",
+ "operator": "=", "value": true}`. The `<>` operator also matches
+ traces without a score of that name.
+
+
+ ## Filter Examples
+
+ ```json
+
+ [
+ {
+ "type": "datetime",
+ "column": "timestamp",
+ "operator": ">=",
+ "value": "2024-01-01T00:00:00Z"
+ },
+ {
+ "type": "string",
+ "column": "userId",
+ "operator": "=",
+ "value": "user-123"
+ },
+ {
+ "type": "number",
+ "column": "totalCost",
+ "operator": ">=",
+ "value": 0.01
+ },
+ {
+ "type": "arrayOptions",
+ "column": "tags",
+ "operator": "all of",
+ "value": ["production", "critical"]
+ },
+ {
+ "type": "stringObject",
+ "column": "metadata",
+ "key": "customer_tier",
+ "operator": "=",
+ "value": "enterprise"
+ }
+ ]
+
+ ```
+
+
+ ## Performance Notes
+
+ - Filtering on `userId`, `sessionId`, or `metadata` may enable skip
+ indexes for better query performance
+
+ - Score filters require a join with the scores table and may impact
+ query performance
+ required: false
+ schema:
+ type: string
+ nullable: true
+ responses:
+ '200':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/Traces'
+ '400':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '401':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '403':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '404':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '405':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ security: *ref_0
+ deprecated: true
+ delete:
+ description: Delete multiple traces
+ operationId: trace_deleteMultiple
+ tags:
+ - Trace
+ parameters: []
+ responses:
+ '200':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/DeleteTraceResponse'
+ '400':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '401':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '403':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '404':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '405':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ security: *ref_0
+ requestBody:
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ properties:
+ traceIds:
+ type: array
+ items:
+ type: string
+ description: List of trace IDs to delete
+ required:
+ - traceIds
+ /api/public/unstable/dashboard-widgets:
+ get:
+ description: |-
+ List dashboard widgets in the project, ordered by most recently
+ updated first.
+
+ Responses may include legacy `traces` widgets created before this
+ API existed. New widgets cannot be created with `view: traces`.
+ operationId: unstable_dashboardWidgets_list
+ tags:
+ - UnstableDashboardWidgets
+ parameters:
+ - name: page
+ in: query
+ description: 1-based page number. Defaults to `1`.
+ required: false
+ schema:
+ type: integer
+ nullable: true
+ - name: limit
+ in: query
+ description: Maximum number of items per page. Defaults to `50`.
+ required: false
+ schema:
+ type: integer
+ nullable: true
+ responses:
+ '200':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/unstableDashboardWidgetList'
+ '400':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '401':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '403':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '404':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '405':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '429':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/unstablePublicApiError'
+ '500':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/unstablePublicApiError'
+ security: *ref_0
+ post:
+ description: >-
+ Create a dashboard widget (a standalone chart definition you place on
+
+ any dashboard).
- - `warningCount` (number) - Count of WARNING level observations
- - `errorCount` (number) - Count of ERROR level observations
+ This endpoint creates the widget only; place it on a dashboard via
- - `defaultCount` (number) - Count of DEFAULT level observations
+ `POST /dashboards/{dashboardId}/placements`.
- - `debugCount` (number) - Count of DEBUG level observations
+ Supported views are `observations`, `scores-numeric`, `scores-boolean`,
+ and `scores-categorical`.
- ### Scores (requires join with scores table)
+ The legacy `traces` view is not supported by this unstable API.
- - `scores_avg` (number) - Average of numeric scores (alias:
- `scores`)
+ Widgets are created as v2 internally.
- - `score_categories` (categoryOptions) - Categorical score values
- - `score_booleans` (booleanObject) - Boolean score values. Use `key`
- for the score name and a boolean `value`, e.g. `{"type":
- "booleanObject", "column": "score_booleans", "key": "is_correct",
- "operator": "=", "value": true}`. The `<>` operator also matches
- traces without a score of that name.
+ `chartConfig` is optional and defaults to the plain config for
+ `chartType`; when `chartConfig.type` is given it must match
- ## Filter Examples
+ `chartType`.
- ```json
- [
- {
- "type": "datetime",
- "column": "timestamp",
- "operator": ">=",
- "value": "2024-01-01T00:00:00Z"
- },
- {
- "type": "string",
- "column": "userId",
- "operator": "=",
- "value": "user-123"
- },
- {
- "type": "number",
- "column": "totalCost",
- "operator": ">=",
- "value": 0.01
- },
- {
- "type": "arrayOptions",
- "column": "tags",
- "operator": "all of",
- "value": ["production", "critical"]
- },
- {
- "type": "stringObject",
- "column": "metadata",
- "key": "customer_tier",
- "operator": "=",
- "value": "enterprise"
- }
- ]
+ Unstable API note:
- ```
+ - This surface may evolve while dashboard/widget APIs are being
+ finalized.
+ operationId: unstable_dashboardWidgets_create
+ tags:
+ - UnstableDashboardWidgets
+ parameters: []
+ responses:
+ '200':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/unstableDashboardWidget'
+ '400':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '401':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '403':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '404':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '405':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '429':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/unstablePublicApiError'
+ '500':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/unstablePublicApiError'
+ security: *ref_0
+ requestBody:
+ required: true
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/unstableCreateDashboardWidgetRequest'
+ /api/public/unstable/dashboard-widgets/{widgetId}:
+ get:
+ description: |-
+ Get a dashboard widget by id.
+ The response may use `view: traces` for legacy widgets.
+ operationId: unstable_dashboardWidgets_get
+ tags:
+ - UnstableDashboardWidgets
+ parameters:
+ - name: widgetId
+ in: path
+ required: true
+ schema:
+ type: string
+ responses:
+ '200':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/unstableDashboardWidget'
+ '400':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '401':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '403':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '404':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '405':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '429':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/unstablePublicApiError'
+ '500':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/unstablePublicApiError'
+ security: *ref_0
+ patch:
+ description: |-
+ Update a dashboard widget.
- ## Performance Notes
+ All fields are optional; at least one field is required.
+ Changing `chartType` without sending `chartConfig` resets the config
+ to the new chart type's defaults. When `chartConfig.type` is given
+ it must match the widget's (possibly updated) `chartType`.
- - Filtering on `userId`, `sessionId`, or `metadata` may enable skip
- indexes for better query performance
+ `view` cannot be changed to the legacy `traces` value. Existing
+ `traces` widgets may be updated on other fields.
+ operationId: unstable_dashboardWidgets_update
+ tags:
+ - UnstableDashboardWidgets
+ parameters:
+ - name: widgetId
+ in: path
+ required: true
+ schema:
+ type: string
+ responses:
+ '200':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/unstableDashboardWidget'
+ '400':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '401':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '403':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '404':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '405':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '429':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/unstablePublicApiError'
+ '500':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/unstablePublicApiError'
+ security: *ref_0
+ requestBody:
+ required: true
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/unstableUpdateDashboardWidgetRequest'
+ delete:
+ description: |-
+ Delete a dashboard widget.
- - Score filters require a join with the scores table and may impact
- query performance
- required: false
+ The API returns `409` while the widget is still placed on a dashboard.
+ Remove those placements first.
+ operationId: unstable_dashboardWidgets_delete
+ tags:
+ - UnstableDashboardWidgets
+ parameters:
+ - name: widgetId
+ in: path
+ required: true
schema:
type: string
+ responses:
+ '200':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/unstableDeleteDashboardWidgetResponse'
+ '400':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '401':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '403':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '404':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '405':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '409':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/unstablePublicApiError'
+ '429':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/unstablePublicApiError'
+ '500':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/unstablePublicApiError'
+ security: *ref_0
+ /api/public/unstable/dashboards:
+ get:
+ description: |-
+ List dashboards in the project, ordered by most recently updated
+ first.
+ operationId: unstable_dashboards_list
+ tags:
+ - UnstableDashboards
+ parameters:
+ - name: page
+ in: query
+ description: 1-based page number. Defaults to `1`.
+ required: false
+ schema:
+ type: integer
+ nullable: true
+ - name: limit
+ in: query
+ description: Maximum number of items per page. Defaults to `50`.
+ required: false
+ schema:
+ type: integer
nullable: true
responses:
'200':
@@ -6798,7 +8083,7 @@ paths:
content:
application/json:
schema:
- $ref: '#/components/schemas/Traces'
+ $ref: '#/components/schemas/unstableDashboardList'
'400':
description: ''
content:
@@ -6824,13 +8109,24 @@ paths:
content:
application/json:
schema: {}
+ '429':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/unstablePublicApiError'
+ '500':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/unstablePublicApiError'
security: *ref_0
- deprecated: true
- delete:
- description: Delete multiple traces
- operationId: trace_deleteMultiple
+ post:
+ description: Create a dashboard.
+ operationId: unstable_dashboards_create
tags:
- - Trace
+ - UnstableDashboards
parameters: []
responses:
'200':
@@ -6838,7 +8134,7 @@ paths:
content:
application/json:
schema:
- $ref: '#/components/schemas/DeleteTraceResponse'
+ $ref: '#/components/schemas/unstableDashboard'
'400':
description: ''
content:
@@ -6864,54 +8160,44 @@ paths:
content:
application/json:
schema: {}
+ '429':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/unstablePublicApiError'
+ '500':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/unstablePublicApiError'
security: *ref_0
requestBody:
required: true
content:
application/json:
schema:
- type: object
- properties:
- traceIds:
- type: array
- items:
- type: string
- description: List of trace IDs to delete
- required:
- - traceIds
- /api/public/unstable/dashboard-widgets:
+ $ref: '#/components/schemas/unstableCreateDashboardRequest'
+ /api/public/unstable/dashboards/{dashboardId}:
get:
- description: |-
- List dashboard widgets in the project, ordered by most recently
- updated first.
-
- Responses may include legacy `traces` widgets created before this
- API existed. New widgets cannot be created with `view: traces`.
- operationId: unstable_dashboardWidgets_list
+ description: Get a dashboard by id.
+ operationId: unstable_dashboards_get
tags:
- - UnstableDashboardWidgets
+ - UnstableDashboards
parameters:
- - name: page
- in: query
- description: 1-based page number. Defaults to `1`.
- required: false
- schema:
- type: integer
- nullable: true
- - name: limit
- in: query
- description: Maximum number of items per page. Defaults to `50`.
- required: false
+ - name: dashboardId
+ in: path
+ required: true
schema:
- type: integer
- nullable: true
+ type: string
responses:
'200':
description: ''
content:
application/json:
schema:
- $ref: '#/components/schemas/unstableDashboardWidgetList'
+ $ref: '#/components/schemas/unstableDashboard'
'400':
description: ''
content:
@@ -6950,48 +8236,86 @@ paths:
schema:
$ref: '#/components/schemas/unstablePublicApiError'
security: *ref_0
- post:
- description: >-
- Create a dashboard widget (a standalone chart definition you place on
-
- any dashboard).
-
-
- This endpoint creates the widget only; place it on a dashboard via
-
- `POST /dashboards/{dashboardId}/placements`.
-
-
- Supported views are `observations`, `scores-numeric`, `scores-boolean`,
- and `scores-categorical`.
-
- The legacy `traces` view is not supported by this unstable API.
-
- Widgets are created as v2 internally.
-
-
- `chartConfig` is optional and defaults to the plain config for
-
- `chartType`; when `chartConfig.type` is given it must match
-
- `chartType`.
-
-
- Unstable API note:
-
- - This surface may evolve while dashboard/widget APIs are being
- finalized.
- operationId: unstable_dashboardWidgets_create
+ patch:
+ description: Update a dashboard's name, description, definition, or filters.
+ operationId: unstable_dashboards_update
+ tags:
+ - UnstableDashboards
+ parameters:
+ - name: dashboardId
+ in: path
+ required: true
+ schema:
+ type: string
+ responses:
+ '200':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/unstableDashboard'
+ '400':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '401':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '403':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '404':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '405':
+ description: ''
+ content:
+ application/json:
+ schema: {}
+ '429':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/unstablePublicApiError'
+ '500':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/unstablePublicApiError'
+ security: *ref_0
+ requestBody:
+ required: true
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/unstableUpdateDashboardRequest'
+ delete:
+ description: Delete a dashboard.
+ operationId: unstable_dashboards_delete
tags:
- - UnstableDashboardWidgets
- parameters: []
+ - UnstableDashboards
+ parameters:
+ - name: dashboardId
+ in: path
+ required: true
+ schema:
+ type: string
responses:
'200':
description: ''
content:
application/json:
schema:
- $ref: '#/components/schemas/unstableDashboardWidget'
+ $ref: '#/components/schemas/unstableDeleteDashboardResponse'
'400':
description: ''
content:
@@ -7030,23 +8354,24 @@ paths:
schema:
$ref: '#/components/schemas/unstablePublicApiError'
security: *ref_0
- requestBody:
- required: true
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/unstableCreateDashboardWidgetRequest'
- /api/public/unstable/dashboard-widgets/{widgetId}:
- get:
+ /api/public/unstable/dashboards/{dashboardId}/placements:
+ post:
description: |-
- Get a dashboard widget by id.
+ Add a placement to a dashboard grid (see `DashboardPlacement` for
+ grid semantics).
- The response may use `view: traces` for legacy widgets.
- operationId: unstable_dashboardWidgets_get
+ `id` and the position fields are optional: when omitted, the
+ placement gets a server-generated id and is appended below all
+ existing tiles as a 6x6 tile. Returns the created placement.
+
+ The referenced widget must exist in the same project or be a
+ Langfuse-managed widget. The API returns `409` if a placement with
+ the same `id` already exists on the dashboard.
+ operationId: unstable_dashboards_addPlacement
tags:
- - UnstableDashboardWidgets
+ - UnstableDashboards
parameters:
- - name: widgetId
+ - name: dashboardId
in: path
required: true
schema:
@@ -7057,7 +8382,7 @@ paths:
content:
application/json:
schema:
- $ref: '#/components/schemas/unstableDashboardWidget'
+ $ref: '#/components/schemas/unstableDashboardPlacement'
'400':
description: ''
content:
@@ -7083,6 +8408,12 @@ paths:
content:
application/json:
schema: {}
+ '409':
+ description: ''
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/unstablePublicApiError'
'429':
description: ''
content:
@@ -7096,22 +8427,29 @@ paths:
schema:
$ref: '#/components/schemas/unstablePublicApiError'
security: *ref_0
+ requestBody:
+ required: true
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/unstableCreateDashboardPlacementRequest'
+ /api/public/unstable/dashboards/{dashboardId}/placements/{placementId}:
patch:
description: |-
- Update a dashboard widget.
-
- All fields are optional; at least one field is required.
- Changing `chartType` without sending `chartConfig` resets the config
- to the new chart type's defaults. When `chartConfig.type` is given
- it must match the widget's (possibly updated) `chartType`.
-
- `view` cannot be changed to the legacy `traces` value. Existing
- `traces` widgets may be updated on other fields.
- operationId: unstable_dashboardWidgets_update
+ Move or resize a placement. All fields are optional; at least one is
+ required. Omitted fields keep their current value. The placement's
+ content (widget/preset reference) and id cannot change — delete and
+ re-add the placement to swap content. Returns the updated placement.
+ operationId: unstable_dashboards_updatePlacement
tags:
- - UnstableDashboardWidgets
+ - UnstableDashboards
parameters:
- - name: widgetId
+ - name: dashboardId
+ in: path
+ required: true
+ schema:
+ type: string
+ - name: placementId
in: path
required: true
schema:
@@ -7122,7 +8460,7 @@ paths:
content:
application/json:
schema:
- $ref: '#/components/schemas/unstableDashboardWidget'
+ $ref: '#/components/schemas/unstableDashboardPlacement'
'400':
description: ''
content:
@@ -7166,18 +8504,21 @@ paths:
content:
application/json:
schema:
- $ref: '#/components/schemas/unstableUpdateDashboardWidgetRequest'
+ $ref: '#/components/schemas/unstableUpdateDashboardPlacementRequest'
delete:
- description: |-
- Delete a dashboard widget.
-
- The API returns `409` while the widget is still placed on a dashboard.
- Remove those placements first.
- operationId: unstable_dashboardWidgets_delete
+ description: >-
+ Remove a placement from a dashboard grid without deleting the referenced
+ widget.
+ operationId: unstable_dashboards_deletePlacement
tags:
- - UnstableDashboardWidgets
+ - UnstableDashboards
parameters:
- - name: widgetId
+ - name: dashboardId
+ in: path
+ required: true
+ schema:
+ type: string
+ - name: placementId
in: path
required: true
schema:
@@ -7188,7 +8529,7 @@ paths:
content:
application/json:
schema:
- $ref: '#/components/schemas/unstableDeleteDashboardWidgetResponse'
+ $ref: '#/components/schemas/unstableDeleteDashboardPlacementResponse'
'400':
description: ''
content:
@@ -7214,12 +8555,6 @@ paths:
content:
application/json:
schema: {}
- '409':
- description: ''
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/unstablePublicApiError'
'429':
description: ''
content:
@@ -7233,36 +8568,123 @@ paths:
schema:
$ref: '#/components/schemas/unstablePublicApiError'
security: *ref_0
- /api/public/unstable/dashboards:
- get:
- description: |-
- List dashboards in the project, ordered by most recently updated
- first.
- operationId: unstable_dashboards_list
- tags:
- - UnstableDashboards
- parameters:
- - name: page
- in: query
- description: 1-based page number. Defaults to `1`.
- required: false
- schema:
- type: integer
- nullable: true
- - name: limit
- in: query
- description: Maximum number of items per page. Defaults to `50`.
- required: false
- schema:
- type: integer
- nullable: true
+ /api/public/unstable/evaluation-rules:
+ post:
+ description: >-
+ **Deprecated:** On Langfuse Cloud, this unstable endpoint is deprecated
+ and will be removed on November 16, 2026. Use the stable
+ `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments
+ are unaffected by this date; the endpoint becomes unavailable when they
+ upgrade to Langfuse v4. See the [Langfuse v3 to v4 upgrade
+ guide](https://langfuse.com/self-hosting/upgrade/upgrade-guides/upgrade-v3-to-v4).
+
+
+ Create an evaluation rule.
+
+
+ An evaluation rule defines **what** incoming data should be evaluated
+ and **how prompt variables should be populated** from that data.
+
+
+ Use this resource after choosing an evaluator from the evaluator
+ endpoints.
+
+
+ Key rules:
+
+ - `name` must be unique within the project for public evaluation rules
+
+ - `target` must be `observation` or `experiment`
+
+ - provide either the compatibility `evaluator` field or the new
+ `evaluators` array, never both
+
+ - each evaluator `name` + `type` must identify an existing evaluator
+ family returned by the evaluator endpoints
+
+ - Langfuse resolves that family to its latest version before saving the
+ evaluation rule
+
+ - for `target=experiment`, use dataset `id` values from `GET
+ /api/public/v2/datasets` when filtering by `datasetId`
+
+ - an omitted LLM-as-judge assignment mapping inherits the evaluator
+ version's default mapping
+
+ - the effective mapping must map every evaluator prompt variable exactly
+ once
+
+ - for `code` evaluators, Langfuse uses the fixed code runtime mapping;
+ omit `mapping` in create and update requests
+
+ - for user-provided `llm_as_judge` mappings, `expected_output` and
+ `experiment_item_metadata` are only valid for `target=experiment`
+
+ - if `enabled=true`, Langfuse validates that the referenced evaluator
+ can currently run
+
+ - at most 500 evaluation rules can be effectively active in one project
+ at the same time (enforced identically by the API, the MCP tools, and
+ the app)
+
+
+ If an evaluation rule with the same `name` already exists in the
+ project, the API returns `409`.
+
+ In that case, update the existing resource with `PATCH
+ /api/public/unstable/evaluation-rules/{evaluationRuleId}` instead of
+ creating a second one.
+
+
+ If enabling this resource would exceed the 500-active limit, the API
+ also returns `409`.
+
+ In that case, disable or pause another active evaluation rule before
+ enabling a new one.
+
+
+ Current scope:
+
+ - evaluation rules are live-ingestion rules only
+
+ - they do not trigger historical backfills
+
+
+ Recovery guidance:
+
+ - `400 invalid_filter_value`: fix the filter `column` or `value` using
+ `details.column`, `details.invalidValues`, and `details.allowedValues`
+
+ - `400 invalid_filter_value` with `details.column=datasetId`: call `GET
+ /api/public/v2/datasets`, then retry with dataset `id` values from that
+ response
+
+ - `400 missing_variable_mapping`: for `llm_as_judge` evaluators, fetch
+ the evaluator again and make sure every variable in `variables` appears
+ exactly once in `mapping`
+
+ - `400 duplicate_variable_mapping`: remove repeated mappings for the
+ same variable
+
+ - `400 invalid_variable_mapping`: for `llm_as_judge`, switch to a valid
+ `source` for the selected `target`, or fix the variable name
+
+ - `400 invalid_json_path`: remove or correct the `jsonPath`
+
+ - `422 evaluator_preflight_failed`: the selected evaluator cannot run
+ with the resolved model configuration. Fix the evaluator/default model
+ setup, then retry the create request.
+ operationId: unstable_evaluationRules_create
+ tags:
+ - UnstableEvaluationRules
+ parameters: []
responses:
'200':
description: ''
content:
application/json:
schema:
- $ref: '#/components/schemas/unstableDashboardList'
+ $ref: '#/components/schemas/unstableEvaluationRule'
'400':
description: ''
content:
@@ -7288,57 +8710,18 @@ paths:
content:
application/json:
schema: {}
- '429':
+ '409':
description: ''
content:
application/json:
schema:
$ref: '#/components/schemas/unstablePublicApiError'
- '500':
+ '422':
description: ''
content:
application/json:
schema:
$ref: '#/components/schemas/unstablePublicApiError'
- security: *ref_0
- post:
- description: Create a dashboard.
- operationId: unstable_dashboards_create
- tags:
- - UnstableDashboards
- parameters: []
- responses:
- '200':
- description: ''
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/unstableDashboard'
- '400':
- description: ''
- content:
- application/json:
- schema: {}
- '401':
- description: ''
- content:
- application/json:
- schema: {}
- '403':
- description: ''
- content:
- application/json:
- schema: {}
- '404':
- description: ''
- content:
- application/json:
- schema: {}
- '405':
- description: ''
- content:
- application/json:
- schema: {}
'429':
description: ''
content:
@@ -7357,82 +8740,50 @@ paths:
content:
application/json:
schema:
- $ref: '#/components/schemas/unstableCreateDashboardRequest'
- /api/public/unstable/dashboards/{dashboardId}:
+ $ref: '#/components/schemas/unstableCreateEvaluationRuleRequest'
+ deprecated: true
get:
- description: Get a dashboard by id.
- operationId: unstable_dashboards_get
+ description: >-
+ **Deprecated:** On Langfuse Cloud, this unstable endpoint is deprecated
+ and will be removed on November 16, 2026. Use the stable
+ `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments
+ are unaffected by this date; the endpoint becomes unavailable when they
+ upgrade to Langfuse v4. See the [Langfuse v3 to v4 upgrade
+ guide](https://langfuse.com/self-hosting/upgrade/upgrade-guides/upgrade-v3-to-v4).
+
+
+ List evaluation rules in the authenticated project.
+
+
+ This includes legacy `trace` and `dataset` rules so they can be
+ inspected and migrated to v4 rules. Legacy rules are read-only through
+ this API; create, update, and delete continue to support only
+ `observation` and `experiment` rules.
+ operationId: unstable_evaluationRules_list
tags:
- - UnstableDashboards
+ - UnstableEvaluationRules
parameters:
- - name: dashboardId
- in: path
- required: true
+ - name: page
+ in: query
+ description: 1-based page number. Defaults to `1`.
+ required: false
schema:
- type: string
- responses:
- '200':
- description: ''
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/unstableDashboard'
- '400':
- description: ''
- content:
- application/json:
- schema: {}
- '401':
- description: ''
- content:
- application/json:
- schema: {}
- '403':
- description: ''
- content:
- application/json:
- schema: {}
- '404':
- description: ''
- content:
- application/json:
- schema: {}
- '405':
- description: ''
- content:
- application/json:
- schema: {}
- '429':
- description: ''
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/unstablePublicApiError'
- '500':
- description: ''
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/unstablePublicApiError'
- security: *ref_0
- patch:
- description: Update a dashboard's name, description, definition, or filters.
- operationId: unstable_dashboards_update
- tags:
- - UnstableDashboards
- parameters:
- - name: dashboardId
- in: path
- required: true
+ type: integer
+ nullable: true
+ - name: limit
+ in: query
+ description: Maximum number of items per page. Defaults to `50`.
+ required: false
schema:
- type: string
+ type: integer
+ nullable: true
responses:
'200':
description: ''
content:
application/json:
schema:
- $ref: '#/components/schemas/unstableDashboard'
+ $ref: '#/components/schemas/unstableEvaluationRules'
'400':
description: ''
content:
@@ -7471,20 +8822,33 @@ paths:
schema:
$ref: '#/components/schemas/unstablePublicApiError'
security: *ref_0
- requestBody:
- required: true
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/unstableUpdateDashboardRequest'
- delete:
- description: Delete a dashboard.
- operationId: unstable_dashboards_delete
+ deprecated: true
+ /api/public/unstable/evaluation-rules/{evaluationRuleId}:
+ get:
+ description: >-
+ **Deprecated:** On Langfuse Cloud, this unstable endpoint is deprecated
+ and will be removed on November 16, 2026. Use the stable
+ `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments
+ are unaffected by this date; the endpoint becomes unavailable when they
+ upgrade to Langfuse v4. See the [Langfuse v3 to v4 upgrade
+ guide](https://langfuse.com/self-hosting/upgrade/upgrade-guides/upgrade-v3-to-v4).
+
+
+ Get one evaluation rule by its identifier.
+
+
+ Use this endpoint to inspect the current evaluator, target, mapping,
+ filters, execution timing, and effective runtime status. Legacy `trace`
+ and `dataset` rules are returned for migration and are read-only through
+ this API.
+ operationId: unstable_evaluationRules_get
tags:
- - UnstableDashboards
+ - UnstableEvaluationRules
parameters:
- - name: dashboardId
+ - name: evaluationRuleId
in: path
+ description: >-
+ Evaluation rule identifier returned by the evaluation rule endpoints.
required: true
schema:
type: string
@@ -7494,7 +8858,7 @@ paths:
content:
application/json:
schema:
- $ref: '#/components/schemas/unstableDeleteDashboardResponse'
+ $ref: '#/components/schemas/unstableReadableEvaluationRule'
'400':
description: ''
content:
@@ -7533,25 +8897,73 @@ paths:
schema:
$ref: '#/components/schemas/unstablePublicApiError'
security: *ref_0
- /api/public/unstable/dashboards/{dashboardId}/placements:
- post:
- description: |-
- Add a placement to a dashboard grid (see `DashboardPlacement` for
- grid semantics).
+ deprecated: true
+ patch:
+ description: >-
+ **Deprecated:** On Langfuse Cloud, this unstable endpoint is deprecated
+ and will be removed on November 16, 2026. Use the stable
+ `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments
+ are unaffected by this date; the endpoint becomes unavailable when they
+ upgrade to Langfuse v4. See the [Langfuse v3 to v4 upgrade
+ guide](https://langfuse.com/self-hosting/upgrade/upgrade-guides/upgrade-v3-to-v4).
- `id` and the position fields are optional: when omitted, the
- placement gets a server-generated id and is appended below all
- existing tiles as a 6x6 tile. Returns the created placement.
- The referenced widget must exist in the same project or be a
- Langfuse-managed widget. The API returns `409` if a placement with
- the same `id` already exists on the dashboard.
- operationId: unstable_dashboards_addPlacement
+ Update an evaluation rule.
+
+
+ Typical uses:
+
+ - enable or disable live execution
+
+ - switch to another evaluator
+
+ - adjust sampling
+
+ - change filters
+
+ - update LLM-as-judge variable mappings
+
+
+ Important behavior:
+
+ - provide only the fields you want to change
+
+ - if you provide `evaluator`, Langfuse resolves that evaluator family to
+ its latest version before saving
+
+ - changing `target`, `filter`, or an LLM-as-judge `mapping` must still
+ produce a valid target-specific configuration
+
+ - if you change `target` for an LLM-as-judge rule, also send a
+ compatible `filter` and `mapping` in the same request unless the
+ existing ones are still valid for the new target
+
+ - for `code` evaluator rules, omit `mapping`; Langfuse stores the fixed
+ code runtime mapping automatically
+
+ - if the resulting config is enabled, Langfuse re-validates that the
+ selected evaluator can run
+
+ - if the update would move a non-active evaluation rule into the active
+ state and the project already has 500 active evaluation rules, the API
+ returns `409`
+
+
+ Recovery guidance:
+
+ - if an LLM-as-judge update fails with `missing_variable_mapping` or
+ `invalid_variable_mapping` after changing `evaluator` or `target`,
+ resend the request with a complete new `mapping`
+
+ - if the update fails with `invalid_filter_value` after changing
+ `target`, resend the request with a target-compatible `filter`
+ operationId: unstable_evaluationRules_update
tags:
- - UnstableDashboards
+ - UnstableEvaluationRules
parameters:
- - name: dashboardId
+ - name: evaluationRuleId
in: path
+ description: Evaluation rule identifier.
required: true
schema:
type: string
@@ -7561,7 +8973,7 @@ paths:
content:
application/json:
schema:
- $ref: '#/components/schemas/unstableDashboardPlacement'
+ $ref: '#/components/schemas/unstableEvaluationRule'
'400':
description: ''
content:
@@ -7587,84 +8999,12 @@ paths:
content:
application/json:
schema: {}
- '409':
- description: ''
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/unstablePublicApiError'
- '429':
- description: ''
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/unstablePublicApiError'
- '500':
+ '422':
description: ''
content:
application/json:
schema:
$ref: '#/components/schemas/unstablePublicApiError'
- security: *ref_0
- requestBody:
- required: true
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/unstableCreateDashboardPlacementRequest'
- /api/public/unstable/dashboards/{dashboardId}/placements/{placementId}:
- patch:
- description: |-
- Move or resize a placement. All fields are optional; at least one is
- required. Omitted fields keep their current value. The placement's
- content (widget/preset reference) and id cannot change — delete and
- re-add the placement to swap content. Returns the updated placement.
- operationId: unstable_dashboards_updatePlacement
- tags:
- - UnstableDashboards
- parameters:
- - name: dashboardId
- in: path
- required: true
- schema:
- type: string
- - name: placementId
- in: path
- required: true
- schema:
- type: string
- responses:
- '200':
- description: ''
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/unstableDashboardPlacement'
- '400':
- description: ''
- content:
- application/json:
- schema: {}
- '401':
- description: ''
- content:
- application/json:
- schema: {}
- '403':
- description: ''
- content:
- application/json:
- schema: {}
- '404':
- description: ''
- content:
- application/json:
- schema: {}
- '405':
- description: ''
- content:
- application/json:
- schema: {}
'429':
description: ''
content:
@@ -7683,22 +9023,30 @@ paths:
content:
application/json:
schema:
- $ref: '#/components/schemas/unstableUpdateDashboardPlacementRequest'
+ $ref: '#/components/schemas/unstableUpdateEvaluationRuleRequest'
+ deprecated: true
delete:
description: >-
- Remove a placement from a dashboard grid without deleting the referenced
- widget.
- operationId: unstable_dashboards_deletePlacement
+ **Deprecated:** On Langfuse Cloud, this unstable endpoint is deprecated
+ and will be removed on November 16, 2026. Use the stable
+ `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments
+ are unaffected by this date; the endpoint becomes unavailable when they
+ upgrade to Langfuse v4. See the [Langfuse v3 to v4 upgrade
+ guide](https://langfuse.com/self-hosting/upgrade/upgrade-guides/upgrade-v3-to-v4).
+
+
+ Delete an evaluation rule.
+
+
+ This removes the live-ingestion rule only. It does not delete the
+ referenced evaluator.
+ operationId: unstable_evaluationRules_delete
tags:
- - UnstableDashboards
+ - UnstableEvaluationRules
parameters:
- - name: dashboardId
- in: path
- required: true
- schema:
- type: string
- - name: placementId
+ - name: evaluationRuleId
in: path
+ description: Evaluation rule identifier.
required: true
schema:
type: string
@@ -7708,7 +9056,7 @@ paths:
content:
application/json:
schema:
- $ref: '#/components/schemas/unstableDeleteDashboardPlacementResponse'
+ $ref: '#/components/schemas/unstableDeleteEvaluationRuleResponse'
'400':
description: ''
content:
@@ -7747,107 +9095,94 @@ paths:
schema:
$ref: '#/components/schemas/unstablePublicApiError'
security: *ref_0
- /api/public/unstable/evaluation-rules:
+ deprecated: true
+ /api/public/unstable/evaluators:
post:
description: >-
- Create an evaluation rule.
-
-
- An evaluation rule defines **what** incoming data should be evaluated
- and **how prompt variables should be populated** from that data.
-
-
- Use this resource after choosing an evaluator from the evaluator
- endpoints.
-
-
- Key rules:
+ **Deprecated:** On Langfuse Cloud, this unstable endpoint is deprecated
+ and will be removed on November 16, 2026. Use the stable
+ `/api/public/v2/evaluators` API instead. Self-hosted deployments are
+ unaffected by this date; the endpoint becomes unavailable when they
+ upgrade to Langfuse v4. See the [Langfuse v3 to v4 upgrade
+ guide](https://langfuse.com/self-hosting/upgrade/upgrade-guides/upgrade-v3-to-v4).
- - `name` must be unique within the project for public evaluation rules
- - `target` must be `observation` or `experiment`
+ Create an evaluator in the authenticated project.
- - provide either the compatibility `evaluator` field or the new
- `evaluators` array, never both
- - each evaluator `name` + `type` must identify an existing evaluator
- family returned by the evaluator endpoints
+ Use evaluators to define **how** Langfuse should score data.
- - Langfuse resolves that family to its latest version before saving the
- evaluation rule
+ LLM-as-a-judge evaluators define a prompt, expected structured output,
+ and optional model configuration.
- - for `target=experiment`, use dataset `id` values from `GET
- /api/public/v2/datasets` when filtering by `datasetId`
+ Code evaluators define source code and a runtime language.
- - an omitted LLM-as-judge assignment mapping inherits the evaluator
- version's default mapping
- - the effective mapping must map every evaluator prompt variable exactly
- once
+ Naming behavior:
- - for `code` evaluators, Langfuse uses the fixed code runtime mapping;
- omit `mapping` in create and update requests
+ - If this is a new evaluator name in your project, Langfuse creates
+ version `1`.
- - for user-provided `llm_as_judge` mappings, `expected_output` and
- `experiment_item_metadata` are only valid for `target=experiment`
+ - If the name already exists in your project, Langfuse creates the next
+ version and returns it.
- - if `enabled=true`, Langfuse validates that the referenced evaluator
- can currently run
+ - The evaluator `id` remains stable across versions.
- - at most 500 evaluation rules can be effectively active in one project
- at the same time (enforced identically by the API, the MCP tools, and
- the app)
+ - Existing evaluation rules automatically use the latest evaluator
+ version; no rule update is required.
- If an evaluation rule with the same `name` already exists in the
- project, the API returns `409`.
+ Recommended workflow:
- In that case, update the existing resource with `PATCH
- /api/public/unstable/evaluation-rules/{evaluationRuleId}` instead of
- creating a second one.
+ 1. Create the evaluator.
+ 2. Read the returned `variables` array.
- If enabling this resource would exceed the 500-active limit, the API
- also returns `409`.
+ 3. Read the returned `outputDefinition.dataType` so the client knows
+ whether future scores will be numeric, boolean, or categorical.
- In that case, disable or pause another active evaluation rule before
- enabling a new one.
+ 4. Create one or more evaluation rules that reference the returned
+ evaluator family using `name` and `type`.
- Current scope:
+ Code evaluator validation:
- - evaluation rules are live-ingestion rules only
+ - At creation, Langfuse only validates the request shape
- - they do not trigger historical backfills
+ - The `sourceCode` itself is not executed here. It is first run
+ (preflight-tested against a sample observation) when you link the
+ evaluator to an evaluation rule, so runtime errors in the code surface
+ at evaluation-rule creation, not at evaluator creation.
Recovery guidance:
- - `400 invalid_filter_value`: fix the filter `column` or `value` using
- `details.column`, `details.invalidValues`, and `details.allowedValues`
+ - `422` with `code=evaluator_preflight_failed`: the evaluator cannot run
+ with the resolved model configuration. Add a valid explicit
+ `modelConfig`, or configure the project's default evaluation model, then
+ retry the same request.
- - `400 invalid_filter_value` with `details.column=datasetId`: call `GET
- /api/public/v2/datasets`, then retry with dataset `id` values from that
- response
+ - `400` with `code=invalid_body`: the request shape is malformed. Use
+ the structured `details.issues` array to fix the specific fields and
+ retry.
- - `400 missing_variable_mapping`: for `llm_as_judge` evaluators, fetch
- the evaluator again and make sure every variable in `variables` appears
- exactly once in `mapping`
+ - `400` with `code=invalid_body` on `outputDefinition`: for
+ `type=llm_as_judge`, send `dataType`, `reasoning.description`, and
+ `score.description`. Do not send `version`; it is not part of the public
+ request shape.
- - `400 duplicate_variable_mapping`: remove repeated mappings for the
- same variable
+ - If `type` is omitted, Langfuse treats the request as
+ `type=llm_as_judge` for backwards compatibility. New clients should send
+ `type` explicitly.
- - `400 invalid_variable_mapping`: for `llm_as_judge`, switch to a valid
- `source` for the selected `target`, or fix the variable name
- - `400 invalid_json_path`: remove or correct the `jsonPath`
+ Unstable API note:
- - `422 evaluator_preflight_failed`: the selected evaluator cannot run
- with the resolved model configuration. Fix the evaluator/default model
- setup, then retry the create request.
- operationId: unstable_evaluationRules_create
+ - This surface may evolve while the underlying evaluation data model is
+ being redesigned.
+ operationId: unstable_evaluators_create
tags:
- - UnstableEvaluationRules
+ - UnstableEvaluators
parameters: []
responses:
'200':
@@ -7855,7 +9190,7 @@ paths:
content:
application/json:
schema:
- $ref: '#/components/schemas/unstableEvaluationRule'
+ $ref: '#/components/schemas/unstableEvaluator'
'400':
description: ''
content:
@@ -7911,19 +9246,20 @@ paths:
content:
application/json:
schema:
- $ref: '#/components/schemas/unstableCreateEvaluationRuleRequest'
+ $ref: '#/components/schemas/unstableCreateEvaluatorRequest'
+ deprecated: true
get:
- description: >-
- List evaluation rules in the authenticated project.
+ description: |-
+ **Deprecated:** On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluators` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4. See the [Langfuse v3 to v4 upgrade guide](https://langfuse.com/self-hosting/upgrade/upgrade-guides/upgrade-v3-to-v4).
+ List the evaluators available to the authenticated project.
- This includes legacy `trace` and `dataset` rules so they can be
- inspected and migrated to v4 rules. Legacy rules are read-only through
- this API; create, update, and delete continue to support only
- `observation` and `experiment` rules.
- operationId: unstable_evaluationRules_list
+ Important behavior:
+ - This endpoint returns the latest version of each available evaluator.
+ - Every evaluator is owned by the authenticated project.
+ operationId: unstable_evaluators_list
tags:
- - UnstableEvaluationRules
+ - UnstableEvaluators
parameters:
- name: page
in: query
@@ -7945,7 +9281,7 @@ paths:
content:
application/json:
schema:
- $ref: '#/components/schemas/unstableEvaluationRules'
+ $ref: '#/components/schemas/unstableEvaluators'
'400':
description: ''
content:
@@ -7984,24 +9320,32 @@ paths:
schema:
$ref: '#/components/schemas/unstablePublicApiError'
security: *ref_0
- /api/public/unstable/evaluation-rules/{evaluationRuleId}:
+ deprecated: true
+ /api/public/unstable/evaluators/{evaluatorId}:
get:
description: >-
- Get one evaluation rule by its identifier.
-
+ **Deprecated:** On Langfuse Cloud, this unstable endpoint is deprecated
+ and will be removed on November 16, 2026. Use the stable
+ `/api/public/v2/evaluators` API instead. Self-hosted deployments are
+ unaffected by this date; the endpoint becomes unavailable when they
+ upgrade to Langfuse v4. See the [Langfuse v3 to v4 upgrade
+ guide](https://langfuse.com/self-hosting/upgrade/upgrade-guides/upgrade-v3-to-v4).
- Use this endpoint to inspect the current evaluator, target, mapping,
- filters, execution timing, and effective runtime status. Legacy `trace`
- and `dataset` rules are returned for migration and are read-only through
- this API.
- operationId: unstable_evaluationRules_get
+
+ Get one evaluator by `id`.
+
+
+ This endpoint always returns the evaluator's latest version. Use it when
+ you want the current prompt, output definition, model configuration, and
+ derived variables for the evaluator you plan to use in an evaluation
+ rule.
+ operationId: unstable_evaluators_get
tags:
- - UnstableEvaluationRules
+ - UnstableEvaluators
parameters:
- - name: evaluationRuleId
+ - name: evaluatorId
in: path
- description: >-
- Evaluation rule identifier returned by the evaluation rule endpoints.
+ description: Evaluator identifier returned by the evaluator endpoints.
required: true
schema:
type: string
@@ -8011,7 +9355,7 @@ paths:
content:
application/json:
schema:
- $ref: '#/components/schemas/unstableReadableEvaluationRule'
+ $ref: '#/components/schemas/unstableEvaluator'
'400':
description: ''
content:
@@ -8050,64 +9394,35 @@ paths:
schema:
$ref: '#/components/schemas/unstablePublicApiError'
security: *ref_0
- patch:
+ deprecated: true
+ delete:
description: >-
- Update an evaluation rule.
-
-
- Typical uses:
-
- - enable or disable live execution
-
- - switch to another evaluator
-
- - adjust sampling
+ **Deprecated:** On Langfuse Cloud, this unstable endpoint is deprecated
+ and will be removed on November 16, 2026. Use the stable
+ `/api/public/v2/evaluators` API instead. Self-hosted deployments are
+ unaffected by this date; the endpoint becomes unavailable when they
+ upgrade to Langfuse v4. See the [Langfuse v3 to v4 upgrade
+ guide](https://langfuse.com/self-hosting/upgrade/upgrade-guides/upgrade-v3-to-v4).
- - change filters
- - update LLM-as-judge variable mappings
+ Delete an evaluator.
Important behavior:
- - provide only the fields you want to change
-
- - if you provide `evaluator`, Langfuse resolves that evaluator family to
- its latest version before saving
-
- - changing `target`, `filter`, or an LLM-as-judge `mapping` must still
- produce a valid target-specific configuration
-
- - if you change `target` for an LLM-as-judge rule, also send a
- compatible `filter` and `mapping` in the same request unless the
- existing ones are still valid for the new target
-
- - for `code` evaluator rules, omit `mapping`; Langfuse stores the fixed
- code runtime mapping automatically
-
- - if the resulting config is enabled, Langfuse re-validates that the
- selected evaluator can run
-
- - if the update would move a non-active evaluation rule into the active
- state and the project already has 500 active evaluation rules, the API
- returns `409`
-
-
- Recovery guidance:
+ - This deletes the evaluator including all of its stored versions.
- - if an LLM-as-judge update fails with `missing_variable_mapping` or
- `invalid_variable_mapping` after changing `evaluator` or `target`,
- resend the request with a complete new `mapping`
+ - Evaluation rule assignments referencing the evaluator are also
+ deleted.
- - if the update fails with `invalid_filter_value` after changing
- `target`, resend the request with a target-compatible `filter`
- operationId: unstable_evaluationRules_update
+ - Scores already produced by the evaluator are not deleted.
+ operationId: unstable_evaluators_delete
tags:
- - UnstableEvaluationRules
+ - UnstableEvaluators
parameters:
- - name: evaluationRuleId
+ - name: evaluatorId
in: path
- description: Evaluation rule identifier.
+ description: Evaluator identifier returned by the evaluator endpoints.
required: true
schema:
type: string
@@ -8117,7 +9432,7 @@ paths:
content:
application/json:
schema:
- $ref: '#/components/schemas/unstableEvaluationRule'
+ $ref: '#/components/schemas/unstableDeleteEvaluatorResponse'
'400':
description: ''
content:
@@ -8143,12 +9458,6 @@ paths:
content:
application/json:
schema: {}
- '422':
- description: ''
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/unstablePublicApiError'
'429':
description: ''
content:
@@ -8162,1919 +9471,2974 @@ paths:
schema:
$ref: '#/components/schemas/unstablePublicApiError'
security: *ref_0
- requestBody:
- required: true
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/unstableUpdateEvaluationRuleRequest'
- delete:
+ deprecated: true
+components:
+ schemas:
+ AnnotationQueueStatus:
+ title: AnnotationQueueStatus
+ type: string
+ enum:
+ - PENDING
+ - COMPLETED
+ AnnotationQueueObjectType:
+ title: AnnotationQueueObjectType
+ type: string
+ enum:
+ - TRACE
+ - OBSERVATION
+ - SESSION
+ AnnotationQueue:
+ title: AnnotationQueue
+ type: object
+ properties:
+ id:
+ type: string
+ name:
+ type: string
+ description:
+ type: string
+ nullable: true
+ scoreConfigIds:
+ type: array
+ items:
+ type: string
+ createdAt:
+ type: string
+ format: date-time
+ updatedAt:
+ type: string
+ format: date-time
+ required:
+ - id
+ - name
+ - description
+ - scoreConfigIds
+ - createdAt
+ - updatedAt
+ AnnotationQueueItem:
+ title: AnnotationQueueItem
+ type: object
+ properties:
+ id:
+ type: string
+ queueId:
+ type: string
+ objectId:
+ type: string
+ objectType:
+ $ref: '#/components/schemas/AnnotationQueueObjectType'
+ status:
+ $ref: '#/components/schemas/AnnotationQueueStatus'
+ completedAt:
+ type: string
+ format: date-time
+ nullable: true
+ createdAt:
+ type: string
+ format: date-time
+ updatedAt:
+ type: string
+ format: date-time
+ required:
+ - id
+ - queueId
+ - objectId
+ - objectType
+ - status
+ - createdAt
+ - updatedAt
+ PaginatedAnnotationQueues:
+ title: PaginatedAnnotationQueues
+ type: object
+ properties:
+ data:
+ type: array
+ items:
+ $ref: '#/components/schemas/AnnotationQueue'
+ meta:
+ $ref: '#/components/schemas/utilsMetaResponse'
+ required:
+ - data
+ - meta
+ PaginatedAnnotationQueueItems:
+ title: PaginatedAnnotationQueueItems
+ type: object
+ properties:
+ data:
+ type: array
+ items:
+ $ref: '#/components/schemas/AnnotationQueueItem'
+ meta:
+ $ref: '#/components/schemas/utilsMetaResponse'
+ required:
+ - data
+ - meta
+ CreateAnnotationQueueRequest:
+ title: CreateAnnotationQueueRequest
+ type: object
+ properties:
+ name:
+ type: string
+ description:
+ type: string
+ nullable: true
+ scoreConfigIds:
+ type: array
+ items:
+ type: string
+ required:
+ - name
+ - scoreConfigIds
+ CreateAnnotationQueueItemRequest:
+ title: CreateAnnotationQueueItemRequest
+ type: object
+ properties:
+ objectId:
+ type: string
+ objectType:
+ $ref: '#/components/schemas/AnnotationQueueObjectType'
+ status:
+ $ref: '#/components/schemas/AnnotationQueueStatus'
+ nullable: true
+ description: Defaults to PENDING for new queue items
+ required:
+ - objectId
+ - objectType
+ UpdateAnnotationQueueItemRequest:
+ title: UpdateAnnotationQueueItemRequest
+ type: object
+ properties:
+ status:
+ $ref: '#/components/schemas/AnnotationQueueStatus'
+ nullable: true
+ DeleteAnnotationQueueItemResponse:
+ title: DeleteAnnotationQueueItemResponse
+ type: object
+ properties:
+ success:
+ type: boolean
+ message:
+ type: string
+ required:
+ - success
+ - message
+ AnnotationQueueAssignmentRequest:
+ title: AnnotationQueueAssignmentRequest
+ type: object
+ properties:
+ userId:
+ type: string
+ required:
+ - userId
+ DeleteAnnotationQueueAssignmentResponse:
+ title: DeleteAnnotationQueueAssignmentResponse
+ type: object
+ properties:
+ success:
+ type: boolean
+ required:
+ - success
+ CreateAnnotationQueueAssignmentResponse:
+ title: CreateAnnotationQueueAssignmentResponse
+ type: object
+ properties:
+ userId:
+ type: string
+ queueId:
+ type: string
+ projectId:
+ type: string
+ required:
+ - userId
+ - queueId
+ - projectId
+ BlobStorageIntegrationType:
+ title: BlobStorageIntegrationType
+ type: string
+ enum:
+ - S3
+ - S3_COMPATIBLE
+ - AZURE_BLOB_STORAGE
+ BlobStorageIntegrationFileType:
+ title: BlobStorageIntegrationFileType
+ type: string
+ enum:
+ - JSON
+ - CSV
+ - JSONL
+ - PARQUET
description: >-
- Delete an evaluation rule.
-
-
- This removes the live-ingestion rule only. It does not delete the
- referenced evaluator.
- operationId: unstable_evaluationRules_delete
- tags:
- - UnstableEvaluationRules
- parameters:
- - name: evaluationRuleId
- in: path
- description: Evaluation rule identifier.
- required: true
- schema:
- type: string
- responses:
- '200':
- description: ''
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/unstableDeleteEvaluationRuleResponse'
- '400':
- description: ''
- content:
- application/json:
- schema: {}
- '401':
- description: ''
- content:
- application/json:
- schema: {}
- '403':
- description: ''
- content:
- application/json:
- schema: {}
- '404':
- description: ''
- content:
- application/json:
- schema: {}
- '405':
- description: ''
- content:
- application/json:
- schema: {}
- '429':
- description: ''
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/unstablePublicApiError'
- '500':
- description: ''
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/unstablePublicApiError'
- security: *ref_0
- /api/public/unstable/evaluators:
- post:
+ File format for exported data. `PARQUET` is a columnar binary format
+ encoded and compressed by the storage engine; gzip compression does not
+ apply to it. Note that the model-price columns (`input_price`,
+ `output_price`, `total_price`) are not included in Parquet observation
+ exports.
+ BlobStorageIntegrationFileTypeResponse:
+ title: BlobStorageIntegrationFileTypeResponse
+ type: string
+ enum:
+ - JSON
+ - CSV
+ - JSONL
+ - PARQUET
+ description: File type reported for an existing integration.
+ BlobStorageExportMode:
+ title: BlobStorageExportMode
+ type: string
+ enum:
+ - FULL_HISTORY
+ - FROM_TODAY
+ - FROM_CUSTOM_DATE
+ BlobStorageExportFrequency:
+ title: BlobStorageExportFrequency
+ type: string
+ enum:
+ - every_20_minutes
+ - hourly
+ - daily
+ - weekly
+ BlobStorageExportSource:
+ title: BlobStorageExportSource
+ type: string
+ enum:
+ - LEGACY_TRACES_OBSERVATIONS
+ - OBSERVATIONS_V2
+ - LEGACY_TRACES_AND_ENRICHED_OBSERVATIONS
description: >-
- Create an evaluator in the authenticated project.
-
-
- Use evaluators to define **how** Langfuse should score data.
-
- LLM-as-a-judge evaluators define a prompt, expected structured output,
- and optional model configuration.
-
- Code evaluators define source code and a runtime language.
-
-
- Naming behavior:
-
- - If this is a new evaluator name in your project, Langfuse creates
- version `1`.
-
- - If the name already exists in your project, Langfuse creates the next
- version and returns it.
-
- - The evaluator `id` remains stable across versions.
-
- - Existing evaluation rules automatically use the latest evaluator
- version; no rule update is required.
-
-
- Recommended workflow:
-
- 1. Create the evaluator.
-
- 2. Read the returned `variables` array.
-
- 3. Read the returned `outputDefinition.dataType` so the client knows
- whether future scores will be numeric, boolean, or categorical.
-
- 4. Create one or more evaluation rules that reference the returned
- evaluator family using `name` and `type`.
-
-
- Code evaluator validation:
+ What data the integration exports.
- - At creation, Langfuse only validates the request shape
+ - `LEGACY_TRACES_OBSERVATIONS`: traces, observations, and scores tables.
+ Observation columns are controlled by `exportFieldGroups`; field groups
+ without a counterpart in this data model (e.g. `trace_context`) are
+ omitted.
- - The `sourceCode` itself is not executed here. It is first run
- (preflight-tested against a sample observation) when you link the
- evaluator to an evaluation rule, so runtime errors in the code surface
- at evaluation-rule creation, not at evaluator creation.
+ - `OBSERVATIONS_V2`: same data model as the
+ `/api/public/v2/observations` endpoint, plus scores. Columns are
+ controlled by `exportFieldGroups`.
+ - `LEGACY_TRACES_AND_ENRICHED_OBSERVATIONS`: both sets. Observation
+ columns of both portions are controlled by `exportFieldGroups`.
- Recovery guidance:
- - `422` with `code=evaluator_preflight_failed`: the evaluator cannot run
- with the resolved model configuration. Add a valid explicit
- `modelConfig`, or configure the project's default evaluation model, then
- retry the same request.
+ **Note:** which sources a deployment accepts depends on how far it has
+ moved to the v4 data model. `OBSERVATIONS_V2` and the
+ enriched-observations portion of
+ `LEGACY_TRACES_AND_ENRICHED_OBSERVATIONS` read the enriched observations
+ table, so they require a deployment that already populates it.
+ `LEGACY_TRACES_OBSERVATIONS` and the legacy portion of
+ `LEGACY_TRACES_AND_ENRICHED_OBSERVATIONS` read the legacy traces and
+ observations tables, so they require a deployment that still populates
+ those. A deployment part-way through the migration populates both and
+ accepts every source. Selecting a source the deployment cannot serve is
+ rejected with `400`, rather than exporting an empty result. See
+ https://langfuse.com/docs/v4.
+ BlobStorageExportFieldGroup:
+ title: BlobStorageExportFieldGroup
+ type: string
+ enum:
+ - core
+ - basic
+ - time
+ - io
+ - metadata
+ - model
+ - usage
+ - prompt
+ - metrics
+ - tools
+ - trace_context
+ description: >-
+ Field group selecting which observation columns are included in the
+ export. Applies to all export sources; groups without a counterpart in
+ the legacy data model (e.g. `trace_context`) are omitted from the legacy
+ observations export.
+ CreateBlobStorageIntegrationRequest:
+ title: CreateBlobStorageIntegrationRequest
+ type: object
+ properties:
+ projectId:
+ type: string
+ description: ID of the project in which to configure the blob storage integration
+ type:
+ $ref: '#/components/schemas/BlobStorageIntegrationType'
+ bucketName:
+ type: string
+ description: >-
+ Name of the storage bucket. For AZURE_BLOB_STORAGE, must be a valid
+ Azure container name (3-63 chars, lowercase letters, numbers, and
+ hyphens only, must start and end with a letter or number, no
+ consecutive hyphens).
+ endpoint:
+ type: string
+ nullable: true
+ description: Custom endpoint URL (required for S3_COMPATIBLE type)
+ region:
+ type: string
+ description: Storage region
+ accessKeyId:
+ type: string
+ nullable: true
+ description: Access key ID for authentication
+ secretAccessKey:
+ type: string
+ nullable: true
+ description: Secret access key for authentication (will be encrypted when stored)
+ prefix:
+ type: string
+ nullable: true
+ description: >-
+ Path prefix for exported files (must end with forward slash if
+ provided)
+ exportFrequency:
+ $ref: '#/components/schemas/BlobStorageExportFrequency'
+ enabled:
+ type: boolean
+ description: Whether the integration is active
+ forcePathStyle:
+ type: boolean
+ description: Use path-style URLs for S3 requests
+ fileType:
+ $ref: '#/components/schemas/BlobStorageIntegrationFileType'
+ exportMode:
+ $ref: '#/components/schemas/BlobStorageExportMode'
+ exportStartDate:
+ type: string
+ format: date-time
+ nullable: true
+ description: >-
+ Custom start date for exports (required when exportMode is
+ FROM_CUSTOM_DATE). Must not be in the future (27 h tolerance for
+ timezone differences).
+ compressed:
+ type: boolean
+ nullable: true
+ description: >-
+ Enable gzip compression for exported files (.csv.gz, .json.gz,
+ .jsonl.gz). Defaults to true.
+ exportSource:
+ $ref: '#/components/schemas/BlobStorageExportSource'
+ nullable: true
+ description: >-
+ Data to export. When omitted on update, the existing value is
+ preserved. When omitted on create, the default is `OBSERVATIONS_V2`
+ on Langfuse Cloud, and on self-hosted deployments
+ `LEGACY_TRACES_OBSERVATIONS` — or `OBSERVATIONS_V2` where the
+ deployment no longer populates the legacy tables. The default is
+ never a source the deployment cannot serve. Required when
+ `exportFieldGroups` is provided.
- - `400` with `code=invalid_body`: the request shape is malformed. Use
- the structured `details.issues` array to fix the specific fields and
- retry.
- - `400` with `code=invalid_body` on `outputDefinition`: for
- `type=llm_as_judge`, send `dataType`, `reasoning.description`, and
- `score.description`. Do not send `version`; it is not part of the public
- request shape.
+ **Cloud-only project deprecation gate (effective 2026-05-20):** For
+ projects created on or after 2026-05-20 on Langfuse Cloud,
+ `LEGACY_TRACES_OBSERVATIONS` and
+ `LEGACY_TRACES_AND_ENRICHED_OBSERVATIONS` are rejected with HTTP
+ 400. Use `OBSERVATIONS_V2` for all new integrations. Self-hosted
+ deployments are unaffected.
- - If `type` is omitted, Langfuse treats the request as
- `type=llm_as_judge` for backwards compatibility. New clients should send
- `type` explicitly.
+ **Cloud-only integration deprecation gate (effective 2026-06-22):**
+ On Langfuse Cloud, legacy export sources are only accepted for blob
+ storage integrations created before 2026-06-22, regardless of
+ project age. Requests that would create a new integration with
+ `LEGACY_TRACES_OBSERVATIONS` or
+ `LEGACY_TRACES_AND_ENRICHED_OBSERVATIONS` are rejected with HTTP
+ 400. Use `OBSERVATIONS_V2` instead. Self-hosted deployments are
+ unaffected.
+ exportFieldGroups:
+ type: array
+ items:
+ $ref: '#/components/schemas/BlobStorageExportFieldGroup'
+ nullable: true
+ description: >-
+ Field groups to include in each exported observation row. Applies to
+ all export sources; must include `core` if provided. When omitted on
+ create, the column default (all groups) applies. When omitted on
+ update, the existing value is preserved.
- Unstable API note:
- - This surface may evolve while the underlying evaluation data model is
- being redesigned.
- operationId: unstable_evaluators_create
- tags:
- - UnstableEvaluators
- parameters: []
- responses:
- '200':
- description: ''
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/unstableEvaluator'
- '400':
- description: ''
- content:
- application/json:
- schema: {}
- '401':
- description: ''
- content:
- application/json:
- schema: {}
- '403':
- description: ''
- content:
- application/json:
- schema: {}
- '404':
- description: ''
- content:
- application/json:
- schema: {}
- '405':
- description: ''
- content:
- application/json:
- schema: {}
- '409':
- description: ''
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/unstablePublicApiError'
- '422':
- description: ''
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/unstablePublicApiError'
- '429':
- description: ''
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/unstablePublicApiError'
- '500':
- description: ''
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/unstablePublicApiError'
- security: *ref_0
- requestBody:
- required: true
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/unstableCreateEvaluatorRequest'
- get:
- description: |-
- List the evaluators available to the authenticated project.
+ `exportFieldGroups` requires `exportSource` to be provided in the
+ same request.
+ required:
+ - projectId
+ - type
+ - bucketName
+ - region
+ - exportFrequency
+ - enabled
+ - forcePathStyle
+ - fileType
+ - exportMode
+ BlobStorageIntegrationResponse:
+ title: BlobStorageIntegrationResponse
+ type: object
+ properties:
+ id:
+ type: string
+ projectId:
+ type: string
+ type:
+ $ref: '#/components/schemas/BlobStorageIntegrationType'
+ bucketName:
+ type: string
+ endpoint:
+ type: string
+ nullable: true
+ region:
+ type: string
+ accessKeyId:
+ type: string
+ nullable: true
+ prefix:
+ type: string
+ exportFrequency:
+ $ref: '#/components/schemas/BlobStorageExportFrequency'
+ enabled:
+ type: boolean
+ forcePathStyle:
+ type: boolean
+ fileType:
+ $ref: '#/components/schemas/BlobStorageIntegrationFileTypeResponse'
+ exportMode:
+ $ref: '#/components/schemas/BlobStorageExportMode'
+ exportStartDate:
+ type: string
+ format: date-time
+ nullable: true
+ compressed:
+ type: boolean
+ exportSource:
+ $ref: '#/components/schemas/BlobStorageExportSource'
+ exportFieldGroups:
+ type: array
+ items:
+ $ref: '#/components/schemas/BlobStorageExportFieldGroup'
+ nullable: true
+ description: >-
+ Field groups included in each exported observation row. An empty
+ list is treated as all groups during export.
+ nextSyncAt:
+ type: string
+ format: date-time
+ nullable: true
+ lastSyncAt:
+ type: string
+ format: date-time
+ nullable: true
+ lastError:
+ type: string
+ nullable: true
+ lastErrorAt:
+ type: string
+ format: date-time
+ nullable: true
+ createdAt:
+ type: string
+ format: date-time
+ updatedAt:
+ type: string
+ format: date-time
+ required:
+ - id
+ - projectId
+ - type
+ - bucketName
+ - endpoint
+ - region
+ - accessKeyId
+ - prefix
+ - exportFrequency
+ - enabled
+ - forcePathStyle
+ - fileType
+ - exportMode
+ - exportStartDate
+ - compressed
+ - exportSource
+ - exportFieldGroups
+ - nextSyncAt
+ - lastSyncAt
+ - lastError
+ - lastErrorAt
+ - createdAt
+ - updatedAt
+ BlobStorageIntegrationsResponse:
+ title: BlobStorageIntegrationsResponse
+ type: object
+ properties:
+ data:
+ type: array
+ items:
+ $ref: '#/components/schemas/BlobStorageIntegrationResponse'
+ required:
+ - data
+ BlobStorageSyncStatus:
+ title: BlobStorageSyncStatus
+ type: string
+ enum:
+ - idle
+ - running
+ - queued
+ - up_to_date
+ - disabled
+ - error
+ description: >-
+ Sync status of the blob storage integration:
- Important behavior:
- - This endpoint returns the latest version of each available evaluator.
- - Every evaluator is owned by the authenticated project.
- operationId: unstable_evaluators_list
- tags:
- - UnstableEvaluators
- parameters:
- - name: page
- in: query
- description: 1-based page number. Defaults to `1`.
- required: false
- schema:
- type: integer
- nullable: true
- - name: limit
- in: query
- description: Maximum number of items per page. Defaults to `50`.
- required: false
- schema:
+ - `disabled` — integration is not enabled
+
+ - `error` — last export failed (see `lastError` for details)
+
+ - `running` — an export job is currently being processed
+
+ - `queued` — next export is overdue (`nextSyncAt` is in the past) and
+ waiting to be picked up by the worker
+
+ - `idle` — enabled but has never exported yet and no export is queued
+
+ - `up_to_date` — all available data has been exported; next export is
+ scheduled for the future
+
+
+ **ETL usage**: poll this endpoint and check for `up_to_date` status.
+ Compare `lastSyncAt` against your
+
+ ETL bookmark to determine if new data is available. Note that exports
+ run with a 20-minute lag buffer,
+
+ so `lastSyncAt` will always be at least 20 minutes behind real-time.
+ BlobStorageIntegrationStatusResponse:
+ title: BlobStorageIntegrationStatusResponse
+ type: object
+ properties:
+ id:
+ type: string
+ projectId:
+ type: string
+ syncStatus:
+ $ref: '#/components/schemas/BlobStorageSyncStatus'
+ enabled:
+ type: boolean
+ lastSyncAt:
+ type: string
+ format: date-time
+ nullable: true
+ description: >-
+ End of the last successfully exported time window. Compare against
+ your ETL bookmark to determine if new data is available. Null if the
+ integration has never synced.
+ nextSyncAt:
+ type: string
+ format: date-time
+ nullable: true
+ description: When the next export is scheduled. Null if no sync has occurred yet.
+ lastError:
+ type: string
+ nullable: true
+ description: >-
+ Raw error message from the storage provider (S3/Azure/GCS) if the
+ last export failed. Cleared on successful export.
+ lastErrorAt:
+ type: string
+ format: date-time
+ nullable: true
+ description: When the last error occurred. Cleared on successful export.
+ required:
+ - id
+ - projectId
+ - syncStatus
+ - enabled
+ - lastSyncAt
+ - nextSyncAt
+ - lastError
+ - lastErrorAt
+ BlobStorageIntegrationDeletionResponse:
+ title: BlobStorageIntegrationDeletionResponse
+ type: object
+ properties:
+ message:
+ type: string
+ required:
+ - message
+ CreateCommentRequest:
+ title: CreateCommentRequest
+ type: object
+ properties:
+ projectId:
+ type: string
+ description: The id of the project to attach the comment to.
+ objectType:
+ type: string
+ description: >-
+ The type of the object to attach the comment to (trace, observation,
+ session, prompt).
+ objectId:
+ type: string
+ description: >-
+ The id of the object to attach the comment to. If this does not
+ reference a valid existing object, an error will be thrown.
+ content:
+ type: string
+ description: >-
+ The content of the comment. May include markdown. Currently limited
+ to 5000 characters.
+ authorUserId:
+ type: string
+ nullable: true
+ description: >-
+ The id of the user who created the comment. Must be a member of the
+ organization that owns the project, otherwise an error will be
+ thrown.
+ required:
+ - projectId
+ - objectType
+ - objectId
+ - content
+ CreateCommentResponse:
+ title: CreateCommentResponse
+ type: object
+ properties:
+ id:
+ type: string
+ description: The id of the created object in Langfuse
+ required:
+ - id
+ GetCommentsResponse:
+ title: GetCommentsResponse
+ type: object
+ properties:
+ data:
+ type: array
+ items:
+ $ref: '#/components/schemas/Comment'
+ meta:
+ $ref: '#/components/schemas/utilsMetaResponse'
+ required:
+ - data
+ - meta
+ Deprecation:
+ title: Deprecation
+ type: object
+ description: >-
+ Migration signal returned by deprecated endpoints. Optional fields are
+ omitted when they have no value.
+ properties:
+ message:
+ type: string
+ description: >-
+ Human- and agent-readable summary of the deprecation and its
+ replacement.
+ replacement:
+ type: string
+ nullable: true
+ description: >-
+ The replacement endpoint, e.g. "GET /api/public/v2/observations".
+ Omitted when the endpoint is being removed without a direct
+ replacement.
+ docsUrl:
+ type: string
+ nullable: true
+ description: Link to the migration documentation (markdown), when available.
+ sunsetAt:
+ type: string
+ nullable: true
+ description: >-
+ ISO date after which the endpoint may stop working, when a removal
+ date is committed.
+ required:
+ - message
+ Trace:
+ title: Trace
+ type: object
+ properties:
+ id:
+ type: string
+ description: The unique identifier of a trace
+ timestamp:
+ type: string
+ format: date-time
+ description: The timestamp when the trace was created
+ name:
+ type: string
+ nullable: true
+ description: The name of the trace
+ input:
+ nullable: true
+ description: The input data of the trace. Can be any JSON.
+ output:
+ nullable: true
+ description: The output data of the trace. Can be any JSON.
+ sessionId:
+ type: string
+ nullable: true
+ description: The session identifier associated with the trace
+ release:
+ type: string
+ nullable: true
+ description: The release version of the application when the trace was created
+ version:
+ type: string
+ nullable: true
+ description: The version of the trace
+ userId:
+ type: string
+ nullable: true
+ description: The user identifier associated with the trace
+ metadata:
+ nullable: true
+ description: The metadata associated with the trace. Can be any JSON.
+ tags:
+ type: array
+ items:
+ type: string
+ description: The tags associated with the trace.
+ public:
+ type: boolean
+ description: Public traces are accessible via url without login
+ environment:
+ type: string
+ description: >-
+ The environment from which this trace originated. Can be any
+ lowercase alphanumeric string with hyphens and underscores that does
+ not start with 'langfuse'.
+ required:
+ - id
+ - timestamp
+ - name
+ - sessionId
+ - release
+ - version
+ - userId
+ - tags
+ - public
+ - environment
+ TraceWithDetails:
+ title: TraceWithDetails
+ type: object
+ properties:
+ htmlPath:
+ type: string
+ description: Path of trace in Langfuse UI
+ latency:
+ type: number
+ format: double
+ nullable: true
+ description: Latency of trace in seconds
+ totalCost:
+ type: number
+ format: double
+ nullable: true
+ description: Cost of trace in USD
+ observations:
+ type: array
+ items:
+ type: string
+ nullable: true
+ description: List of observation ids
+ scores:
+ type: array
+ items:
+ type: string
+ nullable: true
+ description: List of score ids
+ required:
+ - htmlPath
+ allOf:
+ - $ref: '#/components/schemas/Trace'
+ TraceWithFullDetails:
+ title: TraceWithFullDetails
+ type: object
+ properties:
+ htmlPath:
+ type: string
+ description: Path of trace in Langfuse UI
+ latency:
+ type: number
+ format: double
+ nullable: true
+ description: Latency of trace in seconds
+ totalCost:
+ type: number
+ format: double
+ nullable: true
+ description: Cost of trace in USD
+ observations:
+ type: array
+ items:
+ $ref: '#/components/schemas/ObservationsView'
+ description: List of observations
+ scores:
+ type: array
+ items:
+ $ref: '#/components/schemas/ScoreV1'
+ description: List of scores
+ _deprecation:
+ $ref: '#/components/schemas/Deprecation'
+ nullable: true
+ required:
+ - htmlPath
+ - observations
+ - scores
+ allOf:
+ - $ref: '#/components/schemas/Trace'
+ Session:
+ title: Session
+ type: object
+ properties:
+ id:
+ type: string
+ createdAt:
+ type: string
+ format: date-time
+ projectId:
+ type: string
+ environment:
+ type: string
+ description: The environment from which this session originated.
+ required:
+ - id
+ - createdAt
+ - projectId
+ - environment
+ SessionWithTraces:
+ title: SessionWithTraces
+ type: object
+ properties:
+ traces:
+ type: array
+ items:
+ $ref: '#/components/schemas/Trace'
+ _deprecation:
+ $ref: '#/components/schemas/Deprecation'
+ nullable: true
+ required:
+ - traces
+ allOf:
+ - $ref: '#/components/schemas/Session'
+ Observation:
+ title: Observation
+ type: object
+ properties:
+ id:
+ type: string
+ description: The unique identifier of the observation
+ traceId:
+ type: string
+ nullable: true
+ description: The trace ID associated with the observation
+ type:
+ type: string
+ description: The type of the observation
+ name:
+ type: string
+ nullable: true
+ description: The name of the observation
+ startTime:
+ type: string
+ format: date-time
+ description: The start time of the observation
+ endTime:
+ type: string
+ format: date-time
+ nullable: true
+ description: The end time of the observation.
+ completionStartTime:
+ type: string
+ format: date-time
+ nullable: true
+ description: The completion start time of the observation
+ model:
+ type: string
+ nullable: true
+ description: The model used for the observation
+ modelParameters:
+ description: The parameters of the model used for the observation
+ input:
+ description: The input data of the observation
+ version:
+ type: string
+ nullable: true
+ description: The version of the observation
+ metadata:
+ description: Additional metadata of the observation
+ output:
+ description: The output data of the observation
+ usage:
+ $ref: '#/components/schemas/Usage'
+ description: >-
+ (Deprecated. Use usageDetails and costDetails instead.) The usage
+ data of the observation
+ level:
+ $ref: '#/components/schemas/ObservationLevel'
+ description: The level of the observation
+ statusMessage:
+ type: string
+ nullable: true
+ description: The status message of the observation
+ parentObservationId:
+ type: string
+ nullable: true
+ description: The parent observation ID
+ promptId:
+ type: string
+ nullable: true
+ description: The prompt ID associated with the observation
+ usageDetails:
+ type: object
+ additionalProperties:
type: integer
- nullable: true
- responses:
- '200':
- description: ''
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/unstableEvaluators'
- '400':
- description: ''
- content:
- application/json:
- schema: {}
- '401':
- description: ''
- content:
- application/json:
- schema: {}
- '403':
- description: ''
- content:
- application/json:
- schema: {}
- '404':
- description: ''
- content:
- application/json:
- schema: {}
- '405':
- description: ''
- content:
- application/json:
- schema: {}
- '429':
- description: ''
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/unstablePublicApiError'
- '500':
- description: ''
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/unstablePublicApiError'
- security: *ref_0
- /api/public/unstable/evaluators/{evaluatorId}:
- get:
+ description: >-
+ The usage details of the observation. Key is the name of the usage
+ metric, value is the number of units consumed. The total key is the
+ sum of all (non-total) usage metrics or the total value ingested.
+ costDetails:
+ type: object
+ additionalProperties:
+ type: number
+ format: double
+ description: >-
+ The cost details of the observation. Key is the name of the cost
+ metric, value is the cost in USD. The total key is the sum of all
+ (non-total) cost metrics or the total value ingested.
+ environment:
+ type: string
+ description: >-
+ The environment from which this observation originated. Can be any
+ lowercase alphanumeric string with hyphens and underscores that does
+ not start with 'langfuse'.
+ required:
+ - id
+ - traceId
+ - type
+ - name
+ - startTime
+ - endTime
+ - completionStartTime
+ - model
+ - modelParameters
+ - input
+ - version
+ - metadata
+ - output
+ - usage
+ - level
+ - statusMessage
+ - parentObservationId
+ - promptId
+ - usageDetails
+ - costDetails
+ - environment
+ ObservationsView:
+ title: ObservationsView
+ type: object
+ properties:
+ promptName:
+ type: string
+ nullable: true
+ description: The name of the prompt associated with the observation
+ promptVersion:
+ type: integer
+ nullable: true
+ description: The version of the prompt associated with the observation
+ modelId:
+ type: string
+ nullable: true
+ description: The unique identifier of the model
+ inputPrice:
+ type: number
+ format: double
+ nullable: true
+ description: The price of the input in USD
+ outputPrice:
+ type: number
+ format: double
+ nullable: true
+ description: The price of the output in USD.
+ totalPrice:
+ type: number
+ format: double
+ nullable: true
+ description: The total price in USD.
+ calculatedInputCost:
+ type: number
+ format: double
+ nullable: true
+ description: >-
+ (Deprecated. Use usageDetails and costDetails instead.) The
+ calculated cost of the input in USD
+ calculatedOutputCost:
+ type: number
+ format: double
+ nullable: true
+ description: >-
+ (Deprecated. Use usageDetails and costDetails instead.) The
+ calculated cost of the output in USD
+ calculatedTotalCost:
+ type: number
+ format: double
+ nullable: true
+ description: >-
+ (Deprecated. Use usageDetails and costDetails instead.) The
+ calculated total cost in USD
+ latency:
+ type: number
+ format: double
+ nullable: true
+ description: The latency in seconds.
+ timeToFirstToken:
+ type: number
+ format: double
+ nullable: true
+ description: The time to the first token in seconds
+ required:
+ - promptName
+ - promptVersion
+ - modelId
+ - inputPrice
+ - outputPrice
+ - totalPrice
+ - calculatedInputCost
+ - calculatedOutputCost
+ - calculatedTotalCost
+ - latency
+ - timeToFirstToken
+ allOf:
+ - $ref: '#/components/schemas/Observation'
+ ObservationsViewSingle:
+ title: ObservationsViewSingle
+ type: object
+ properties:
+ _deprecation:
+ $ref: '#/components/schemas/Deprecation'
+ nullable: true
+ allOf:
+ - $ref: '#/components/schemas/ObservationsView'
+ ObservationV2:
+ title: ObservationV2
+ type: object
description: >-
- Get one evaluator by `id`.
+ An observation from the v2 API with field-group-based selection.
+
+ Core fields are always present. Other fields are included only when
+ their field group is requested.
+ properties:
+ id:
+ type: string
+ description: The unique identifier of the observation
+ traceId:
+ type: string
+ nullable: true
+ description: The trace ID associated with the observation
+ startTime:
+ type: string
+ format: date-time
+ description: The start time of the observation
+ endTime:
+ type: string
+ format: date-time
+ nullable: true
+ description: The end time of the observation
+ projectId:
+ type: string
+ description: The project ID this observation belongs to
+ parentObservationId:
+ type: string
+ nullable: true
+ description: >-
+ The physical parent observation ID, if present.
+ Observations marked as app roots by the SDK may retain a non-null
+ parent ID.
+ type:
+ type: string
+ description: The type of the observation (e.g. GENERATION, SPAN, EVENT)
+ isRootObservation:
+ type: boolean
+ nullable: true
+ description: >-
+ Whether this observation is a logical root.
- This endpoint always returns the evaluator's latest version. Use it when
- you want the current prompt, output definition, model configuration, and
- derived variables for the evaluator you plan to use in an evaluation
- rule.
- operationId: unstable_evaluators_get
- tags:
- - UnstableEvaluators
- parameters:
- - name: evaluatorId
- in: path
- description: Evaluator identifier returned by the evaluator endpoints.
- required: true
- schema:
+ This is true for observations without a physical parent and
+ observations marked as app roots by the SDK.
+ name:
+ type: string
+ nullable: true
+ description: The name of the observation
+ level:
+ $ref: '#/components/schemas/ObservationLevel'
+ nullable: true
+ description: The level of the observation
+ statusMessage:
+ type: string
+ nullable: true
+ description: The status message of the observation
+ version:
+ type: string
+ nullable: true
+ description: The version of the observation
+ environment:
+ type: string
+ nullable: true
+ description: The environment from which this observation originated
+ bookmarked:
+ type: boolean
+ nullable: true
+ description: Whether the observation is bookmarked
+ public:
+ type: boolean
+ nullable: true
+ description: Whether the observation is public
+ userId:
+ type: string
+ nullable: true
+ description: The user ID associated with the observation
+ sessionId:
+ type: string
+ nullable: true
+ description: The session ID associated with the observation
+ completionStartTime:
+ type: string
+ format: date-time
+ nullable: true
+ description: The completion start time of the observation
+ createdAt:
+ type: string
+ format: date-time
+ nullable: true
+ description: The creation timestamp of the observation
+ updatedAt:
+ type: string
+ format: date-time
+ nullable: true
+ description: The last update timestamp of the observation
+ input:
+ nullable: true
+ description: The input data of the observation
+ output:
+ nullable: true
+ description: The output data of the observation
+ metadata:
+ nullable: true
+ description: Additional metadata of the observation
+ providedModelName:
+ type: string
+ nullable: true
+ description: The model name as provided by the user
+ internalModelId:
+ type: string
+ nullable: true
+ description: The internal model ID matched by Langfuse
+ modelParameters:
+ nullable: true
+ description: The parameters of the model used for the observation
+ usageDetails:
+ type: object
+ additionalProperties:
+ type: integer
+ nullable: true
+ description: >-
+ The usage details of the observation. Key is the usage metric name,
+ value is the number of units consumed.
+ costDetails:
+ type: object
+ additionalProperties:
+ type: number
+ format: double
+ nullable: true
+ description: >-
+ The cost details of the observation. Key is the cost metric name,
+ value is the cost in USD.
+ totalCost:
+ type: number
+ format: double
+ nullable: true
+ description: The total cost of the observation in USD
+ usagePricingTierName:
+ type: string
+ nullable: true
+ description: >-
+ The name of the pricing tier applied to this observation's usage
+ costs
+ promptId:
+ type: string
+ nullable: true
+ description: The prompt ID associated with the observation
+ promptName:
+ type: string
+ nullable: true
+ description: The prompt name associated with the observation
+ promptVersion:
+ type: integer
+ nullable: true
+ description: The prompt version associated with the observation
+ latency:
+ type: number
+ format: double
+ nullable: true
+ description: The latency in seconds
+ timeToFirstToken:
+ type: number
+ format: double
+ nullable: true
+ description: The time to first token in seconds
+ modelId:
+ type: string
+ nullable: true
+ description: >-
+ The matched model ID. Null when the `model` field group is not
+ requested.
+ inputPrice:
+ type: string
+ nullable: true
+ description: >-
+ The input token price (USD per unit) from the matched model,
+ serialized as a decimal string (e.g. "0.0001"). Null when the
+ `model` field group is not requested.
+ outputPrice:
+ type: string
+ nullable: true
+ description: >-
+ The output token price (USD per unit) from the matched model,
+ serialized as a decimal string (e.g. "0.0001"). Null when the
+ `model` field group is not requested.
+ totalPrice:
+ type: string
+ nullable: true
+ description: >-
+ The total token price (USD per unit) from the matched model,
+ serialized as a decimal string (e.g. "0.0001"). Null when the
+ `model` field group is not requested.
+ traceName:
+ type: string
+ nullable: true
+ description: The name of the parent trace
+ tags:
+ type: array
+ items:
type: string
- responses:
- '200':
- description: ''
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/unstableEvaluator'
- '400':
- description: ''
- content:
- application/json:
- schema: {}
- '401':
- description: ''
- content:
- application/json:
- schema: {}
- '403':
- description: ''
- content:
- application/json:
- schema: {}
- '404':
- description: ''
- content:
- application/json:
- schema: {}
- '405':
- description: ''
- content:
- application/json:
- schema: {}
- '429':
- description: ''
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/unstablePublicApiError'
- '500':
- description: ''
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/unstablePublicApiError'
- security: *ref_0
- delete:
+ nullable: true
+ description: Tags from the parent trace (denormalized onto the observation)
+ release:
+ type: string
+ nullable: true
+ description: The release version of the parent trace
+ required:
+ - id
+ - traceId
+ - startTime
+ - endTime
+ - projectId
+ - parentObservationId
+ - type
+ - modelId
+ - inputPrice
+ - outputPrice
+ - totalPrice
+ Usage:
+ title: Usage
+ type: object
description: >-
- Delete an evaluator.
-
-
- Important behavior:
-
- - This deletes the evaluator including all of its stored versions.
-
- - Evaluation rule assignments referencing the evaluator are also
- deleted.
-
- - Scores already produced by the evaluator are not deleted.
- operationId: unstable_evaluators_delete
- tags:
- - UnstableEvaluators
- parameters:
- - name: evaluatorId
- in: path
- description: Evaluator identifier returned by the evaluator endpoints.
- required: true
- schema:
- type: string
- responses:
- '200':
- description: ''
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/unstableDeleteEvaluatorResponse'
- '400':
- description: ''
- content:
- application/json:
- schema: {}
- '401':
- description: ''
- content:
- application/json:
- schema: {}
- '403':
- description: ''
- content:
- application/json:
- schema: {}
- '404':
- description: ''
- content:
- application/json:
- schema: {}
- '405':
- description: ''
- content:
- application/json:
- schema: {}
- '429':
- description: ''
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/unstablePublicApiError'
- '500':
- description: ''
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/unstablePublicApiError'
- security: *ref_0
-components:
- schemas:
- AnnotationQueueStatus:
- title: AnnotationQueueStatus
- type: string
- enum:
- - PENDING
- - COMPLETED
- AnnotationQueueObjectType:
- title: AnnotationQueueObjectType
- type: string
- enum:
- - TRACE
- - OBSERVATION
- - SESSION
- AnnotationQueue:
- title: AnnotationQueue
+ (Deprecated. Use usageDetails and costDetails instead.) Standard
+ interface for usage and cost
+ properties:
+ input:
+ type: integer
+ description: Number of input units (e.g. tokens)
+ output:
+ type: integer
+ description: Number of output units (e.g. tokens)
+ total:
+ type: integer
+ description: Defaults to input+output if not set
+ unit:
+ type: string
+ nullable: true
+ description: Unit of measurement
+ inputCost:
+ type: number
+ format: double
+ nullable: true
+ description: USD input cost
+ outputCost:
+ type: number
+ format: double
+ nullable: true
+ description: USD output cost
+ totalCost:
+ type: number
+ format: double
+ nullable: true
+ description: USD total cost, defaults to input+output
+ required:
+ - input
+ - output
+ - total
+ - unit
+ ScoreConfig:
+ title: ScoreConfig
type: object
+ description: Configuration for a score
properties:
id:
type: string
name:
type: string
- description:
+ createdAt:
type: string
+ format: date-time
+ updatedAt:
+ type: string
+ format: date-time
+ projectId:
+ type: string
+ dataType:
+ $ref: '#/components/schemas/ScoreConfigDataType'
+ isArchived:
+ type: boolean
+ description: Whether the score config is archived. Defaults to false
+ minValue:
+ type: number
+ format: double
nullable: true
- scoreConfigIds:
+ description: >-
+ Sets minimum value for numerical scores. If not set, the minimum
+ value defaults to -∞
+ maxValue:
+ type: number
+ format: double
+ nullable: true
+ description: >-
+ Sets maximum value for numerical scores. If not set, the maximum
+ value defaults to +∞
+ categories:
type: array
items:
- type: string
- createdAt:
- type: string
- format: date-time
- updatedAt:
+ $ref: '#/components/schemas/ConfigCategory'
+ nullable: true
+ description: Configures custom categories for categorical scores
+ description:
type: string
- format: date-time
+ nullable: true
+ description: Description of the score config
required:
- id
- name
- - description
- - scoreConfigIds
- createdAt
- updatedAt
- AnnotationQueueItem:
- title: AnnotationQueueItem
+ - projectId
+ - dataType
+ - isArchived
+ ConfigCategory:
+ title: ConfigCategory
+ type: object
+ properties:
+ value:
+ type: number
+ format: double
+ label:
+ type: string
+ required:
+ - value
+ - label
+ BaseScoreV1:
+ title: BaseScoreV1
type: object
properties:
id:
type: string
- queueId:
+ traceId:
type: string
- objectId:
+ name:
type: string
- objectType:
- $ref: '#/components/schemas/AnnotationQueueObjectType'
- status:
- $ref: '#/components/schemas/AnnotationQueueStatus'
- completedAt:
+ source:
+ $ref: '#/components/schemas/ScoreSource'
+ observationId:
type: string
- format: date-time
nullable: true
+ description: The observation ID associated with the score
+ timestamp:
+ type: string
+ format: date-time
createdAt:
type: string
format: date-time
updatedAt:
type: string
format: date-time
+ authorUserId:
+ type: string
+ nullable: true
+ description: The user ID of the author
+ comment:
+ type: string
+ nullable: true
+ description: Comment on the score
+ metadata:
+ description: Metadata associated with the score
+ configId:
+ type: string
+ nullable: true
+ description: >-
+ Reference a score config on a score. When set, config and score name
+ must be equal and value must comply to optionally defined numerical
+ range
+ queueId:
+ type: string
+ nullable: true
+ description: >-
+ The annotation queue referenced by the score. Indicates if score was
+ initially created while processing annotation queue.
+ environment:
+ type: string
+ description: >-
+ The environment from which this score originated. Can be any
+ lowercase alphanumeric string with hyphens and underscores that does
+ not start with 'langfuse'.
required:
- id
- - queueId
- - objectId
- - objectType
- - status
+ - traceId
+ - name
+ - source
+ - timestamp
- createdAt
- updatedAt
- PaginatedAnnotationQueues:
- title: PaginatedAnnotationQueues
+ - authorUserId
+ - comment
+ - metadata
+ - configId
+ - queueId
+ - environment
+ NumericScoreV1:
+ title: NumericScoreV1
type: object
properties:
- data:
- type: array
- items:
- $ref: '#/components/schemas/AnnotationQueue'
- meta:
- $ref: '#/components/schemas/utilsMetaResponse'
+ value:
+ type: number
+ format: double
+ description: The numeric value of the score
required:
- - data
- - meta
- PaginatedAnnotationQueueItems:
- title: PaginatedAnnotationQueueItems
+ - value
+ allOf:
+ - $ref: '#/components/schemas/BaseScoreV1'
+ BooleanScoreV1:
+ title: BooleanScoreV1
type: object
properties:
- data:
- type: array
- items:
- $ref: '#/components/schemas/AnnotationQueueItem'
- meta:
- $ref: '#/components/schemas/utilsMetaResponse'
+ value:
+ type: number
+ format: double
+ description: >-
+ The numeric value of the score. Equals 1 for "True" and 0 for "False"
+ stringValue:
+ type: string
+ description: >-
+ The string representation of the score value. Is inferred from the
+ numeric value and equals "True" or "False"
required:
- - data
- - meta
- CreateAnnotationQueueRequest:
- title: CreateAnnotationQueueRequest
+ - value
+ - stringValue
+ allOf:
+ - $ref: '#/components/schemas/BaseScoreV1'
+ CategoricalScoreV1:
+ title: CategoricalScoreV1
+ type: object
+ properties:
+ value:
+ type: number
+ format: double
+ description: >-
+ Represents the numeric category mapping of the stringValue. If no
+ config is linked, defaults to 0.
+ stringValue:
+ type: string
+ description: >-
+ The string representation of the score value. If no config is
+ linked, can be any string. Otherwise, must map to a config category
+ required:
+ - value
+ - stringValue
+ allOf:
+ - $ref: '#/components/schemas/BaseScoreV1'
+ TextScoreV1:
+ title: TextScoreV1
+ type: object
+ properties:
+ stringValue:
+ type: string
+ description: The text content of the score (1-500 characters)
+ required:
+ - stringValue
+ allOf:
+ - $ref: '#/components/schemas/BaseScoreV1'
+ ScoreV1:
+ title: ScoreV1
+ oneOf:
+ - type: object
+ allOf:
+ - type: object
+ properties:
+ dataType:
+ type: string
+ enum:
+ - NUMERIC
+ - $ref: '#/components/schemas/NumericScoreV1'
+ required:
+ - dataType
+ - type: object
+ allOf:
+ - type: object
+ properties:
+ dataType:
+ type: string
+ enum:
+ - CATEGORICAL
+ - $ref: '#/components/schemas/CategoricalScoreV1'
+ required:
+ - dataType
+ - type: object
+ allOf:
+ - type: object
+ properties:
+ dataType:
+ type: string
+ enum:
+ - BOOLEAN
+ - $ref: '#/components/schemas/BooleanScoreV1'
+ required:
+ - dataType
+ - type: object
+ allOf:
+ - type: object
+ properties:
+ dataType:
+ type: string
+ enum:
+ - TEXT
+ - $ref: '#/components/schemas/TextScoreV1'
+ required:
+ - dataType
+ BaseScore:
+ title: BaseScore
type: object
properties:
+ id:
+ type: string
+ traceId:
+ type: string
+ nullable: true
+ description: The trace ID associated with the score
+ sessionId:
+ type: string
+ nullable: true
+ description: The session ID associated with the score
+ observationId:
+ type: string
+ nullable: true
+ description: The observation ID associated with the score
+ datasetRunId:
+ type: string
+ nullable: true
+ description: The dataset run ID associated with the score
name:
type: string
- description:
+ source:
+ $ref: '#/components/schemas/ScoreSource'
+ timestamp:
+ type: string
+ format: date-time
+ createdAt:
+ type: string
+ format: date-time
+ updatedAt:
+ type: string
+ format: date-time
+ authorUserId:
type: string
nullable: true
- scoreConfigIds:
- type: array
- items:
- type: string
- required:
- - name
- - scoreConfigIds
- CreateAnnotationQueueItemRequest:
- title: CreateAnnotationQueueItemRequest
- type: object
- properties:
- objectId:
+ description: The user ID of the author
+ comment:
type: string
- objectType:
- $ref: '#/components/schemas/AnnotationQueueObjectType'
- status:
- $ref: '#/components/schemas/AnnotationQueueStatus'
nullable: true
- description: Defaults to PENDING for new queue items
- required:
- - objectId
- - objectType
- UpdateAnnotationQueueItemRequest:
- title: UpdateAnnotationQueueItemRequest
- type: object
- properties:
- status:
- $ref: '#/components/schemas/AnnotationQueueStatus'
+ description: Comment on the score
+ metadata:
+ description: Metadata associated with the score
+ configId:
+ type: string
nullable: true
- DeleteAnnotationQueueItemResponse:
- title: DeleteAnnotationQueueItemResponse
+ description: >-
+ Reference a score config on a score. When set, config and score name
+ must be equal and value must comply to optionally defined numerical
+ range
+ queueId:
+ type: string
+ nullable: true
+ description: >-
+ The annotation queue referenced by the score. Indicates if score was
+ initially created while processing annotation queue.
+ environment:
+ type: string
+ description: >-
+ The environment from which this score originated. Can be any
+ lowercase alphanumeric string with hyphens and underscores that does
+ not start with 'langfuse'.
+ required:
+ - id
+ - name
+ - source
+ - timestamp
+ - createdAt
+ - updatedAt
+ - authorUserId
+ - comment
+ - metadata
+ - configId
+ - queueId
+ - environment
+ NumericScore:
+ title: NumericScore
type: object
properties:
- success:
- type: boolean
- message:
- type: string
+ value:
+ type: number
+ format: double
+ description: The numeric value of the score
required:
- - success
- - message
- AnnotationQueueAssignmentRequest:
- title: AnnotationQueueAssignmentRequest
+ - value
+ allOf:
+ - $ref: '#/components/schemas/BaseScore'
+ BooleanScore:
+ title: BooleanScore
type: object
properties:
- userId:
+ value:
+ type: number
+ format: double
+ description: >-
+ The numeric value of the score. Equals 1 for "True" and 0 for "False"
+ stringValue:
type: string
+ description: >-
+ The string representation of the score value. Is inferred from the
+ numeric value and equals "True" or "False"
required:
- - userId
- DeleteAnnotationQueueAssignmentResponse:
- title: DeleteAnnotationQueueAssignmentResponse
+ - value
+ - stringValue
+ allOf:
+ - $ref: '#/components/schemas/BaseScore'
+ CategoricalScore:
+ title: CategoricalScore
type: object
properties:
- success:
- type: boolean
+ value:
+ type: number
+ format: double
+ description: >-
+ Represents the numeric category mapping of the stringValue. If no
+ config is linked, defaults to 0.
+ stringValue:
+ type: string
+ description: >-
+ The string representation of the score value. If no config is
+ linked, can be any string. Otherwise, must map to a config category
required:
- - success
- CreateAnnotationQueueAssignmentResponse:
- title: CreateAnnotationQueueAssignmentResponse
+ - value
+ - stringValue
+ allOf:
+ - $ref: '#/components/schemas/BaseScore'
+ CorrectionScore:
+ title: CorrectionScore
type: object
properties:
- userId:
- type: string
- queueId:
- type: string
- projectId:
+ value:
+ type: number
+ format: double
+ description: The numeric value of the score. Always 0 for correction scores.
+ stringValue:
type: string
+ description: The string representation of the correction content
required:
- - userId
- - queueId
- - projectId
- BlobStorageIntegrationType:
- title: BlobStorageIntegrationType
- type: string
- enum:
- - S3
- - S3_COMPATIBLE
- - AZURE_BLOB_STORAGE
- BlobStorageIntegrationFileType:
- title: BlobStorageIntegrationFileType
- type: string
- enum:
- - JSON
- - CSV
- - JSONL
- - PARQUET
- description: >-
- File format for exported data. `PARQUET` is a columnar binary format
- encoded and compressed by the storage engine; gzip compression does not
- apply to it. Note that the model-price columns (`input_price`,
- `output_price`, `total_price`) are not included in Parquet observation
- exports.
- BlobStorageIntegrationFileTypeResponse:
- title: BlobStorageIntegrationFileTypeResponse
- type: string
- enum:
- - JSON
- - CSV
- - JSONL
- - PARQUET
- description: File type reported for an existing integration.
- BlobStorageExportMode:
- title: BlobStorageExportMode
- type: string
- enum:
- - FULL_HISTORY
- - FROM_TODAY
- - FROM_CUSTOM_DATE
- BlobStorageExportFrequency:
- title: BlobStorageExportFrequency
- type: string
- enum:
- - every_20_minutes
- - hourly
- - daily
- - weekly
- BlobStorageExportSource:
- title: BlobStorageExportSource
- type: string
- enum:
- - LEGACY_TRACES_OBSERVATIONS
- - OBSERVATIONS_V2
- - LEGACY_TRACES_AND_ENRICHED_OBSERVATIONS
- description: >-
- What data the integration exports.
-
- - `LEGACY_TRACES_OBSERVATIONS`: traces, observations, and scores tables.
- Observation columns are controlled by `exportFieldGroups`; field groups
- without a counterpart in this data model (e.g. `trace_context`) are
- omitted.
-
- - `OBSERVATIONS_V2`: same data model as the
- `/api/public/v2/observations` endpoint, plus scores. Columns are
- controlled by `exportFieldGroups`.
-
- - `LEGACY_TRACES_AND_ENRICHED_OBSERVATIONS`: both sets. Observation
- columns of both portions are controlled by `exportFieldGroups`.
-
-
- **Note:** which sources a deployment accepts depends on how far it has
- moved to the v4 data model. `OBSERVATIONS_V2` and the
- enriched-observations portion of
- `LEGACY_TRACES_AND_ENRICHED_OBSERVATIONS` read the enriched observations
- table, so they require a deployment that already populates it.
- `LEGACY_TRACES_OBSERVATIONS` and the legacy portion of
- `LEGACY_TRACES_AND_ENRICHED_OBSERVATIONS` read the legacy traces and
- observations tables, so they require a deployment that still populates
- those. A deployment part-way through the migration populates both and
- accepts every source. Selecting a source the deployment cannot serve is
- rejected with `400`, rather than exporting an empty result. See
- https://langfuse.com/docs/v4.
- BlobStorageExportFieldGroup:
- title: BlobStorageExportFieldGroup
- type: string
- enum:
- - core
- - basic
- - time
- - io
- - metadata
- - model
- - usage
- - prompt
- - metrics
- - tools
- - trace_context
- description: >-
- Field group selecting which observation columns are included in the
- export. Applies to all export sources; groups without a counterpart in
- the legacy data model (e.g. `trace_context`) are omitted from the legacy
- observations export.
- CreateBlobStorageIntegrationRequest:
- title: CreateBlobStorageIntegrationRequest
+ - value
+ - stringValue
+ allOf:
+ - $ref: '#/components/schemas/BaseScore'
+ TextScore:
+ title: TextScore
type: object
properties:
- projectId:
+ stringValue:
type: string
- description: ID of the project in which to configure the blob storage integration
- type:
- $ref: '#/components/schemas/BlobStorageIntegrationType'
- bucketName:
+ description: The text content of the score (1-500 characters)
+ required:
+ - stringValue
+ allOf:
+ - $ref: '#/components/schemas/BaseScore'
+ Score:
+ title: Score
+ oneOf:
+ - type: object
+ allOf:
+ - type: object
+ properties:
+ dataType:
+ type: string
+ enum:
+ - NUMERIC
+ - $ref: '#/components/schemas/NumericScore'
+ required:
+ - dataType
+ - type: object
+ allOf:
+ - type: object
+ properties:
+ dataType:
+ type: string
+ enum:
+ - CATEGORICAL
+ - $ref: '#/components/schemas/CategoricalScore'
+ required:
+ - dataType
+ - type: object
+ allOf:
+ - type: object
+ properties:
+ dataType:
+ type: string
+ enum:
+ - BOOLEAN
+ - $ref: '#/components/schemas/BooleanScore'
+ required:
+ - dataType
+ - type: object
+ allOf:
+ - type: object
+ properties:
+ dataType:
+ type: string
+ enum:
+ - CORRECTION
+ - $ref: '#/components/schemas/CorrectionScore'
+ required:
+ - dataType
+ - type: object
+ allOf:
+ - type: object
+ properties:
+ dataType:
+ type: string
+ enum:
+ - TEXT
+ - $ref: '#/components/schemas/TextScore'
+ required:
+ - dataType
+ properties:
+ _deprecation:
+ $ref: '#/components/schemas/Deprecation'
+ nullable: true
+ CreateScoreValue:
+ title: CreateScoreValue
+ oneOf:
+ - type: number
+ format: double
+ - type: string
+ description: >-
+ The value of the score. Must be passed as string for categorical and
+ text scores, and numeric for boolean and numeric scores
+ Comment:
+ title: Comment
+ type: object
+ properties:
+ id:
type: string
- description: >-
- Name of the storage bucket. For AZURE_BLOB_STORAGE, must be a valid
- Azure container name (3-63 chars, lowercase letters, numbers, and
- hyphens only, must start and end with a letter or number, no
- consecutive hyphens).
- endpoint:
+ projectId:
type: string
- nullable: true
- description: Custom endpoint URL (required for S3_COMPATIBLE type)
- region:
+ createdAt:
type: string
- description: Storage region
- accessKeyId:
+ format: date-time
+ updatedAt:
type: string
- nullable: true
- description: Access key ID for authentication
- secretAccessKey:
+ format: date-time
+ objectType:
+ $ref: '#/components/schemas/CommentObjectType'
+ objectId:
type: string
- nullable: true
- description: Secret access key for authentication (will be encrypted when stored)
- prefix:
+ content:
type: string
- nullable: true
- description: >-
- Path prefix for exported files (must end with forward slash if
- provided)
- exportFrequency:
- $ref: '#/components/schemas/BlobStorageExportFrequency'
- enabled:
- type: boolean
- description: Whether the integration is active
- forcePathStyle:
- type: boolean
- description: Use path-style URLs for S3 requests
- fileType:
- $ref: '#/components/schemas/BlobStorageIntegrationFileType'
- exportMode:
- $ref: '#/components/schemas/BlobStorageExportMode'
- exportStartDate:
+ authorUserId:
type: string
- format: date-time
- nullable: true
- description: >-
- Custom start date for exports (required when exportMode is
- FROM_CUSTOM_DATE). Must not be in the future (27 h tolerance for
- timezone differences).
- compressed:
- type: boolean
- nullable: true
- description: >-
- Enable gzip compression for exported files (.csv.gz, .json.gz,
- .jsonl.gz). Defaults to true.
- exportSource:
- $ref: '#/components/schemas/BlobStorageExportSource'
- nullable: true
- description: >-
- Data to export. When omitted on update, the existing value is
- preserved. When omitted on create, the default is `OBSERVATIONS_V2`
- on Langfuse Cloud, and on self-hosted deployments
- `LEGACY_TRACES_OBSERVATIONS` — or `OBSERVATIONS_V2` where the
- deployment no longer populates the legacy tables. The default is
- never a source the deployment cannot serve. Required when
- `exportFieldGroups` is provided.
-
-
- **Cloud-only project deprecation gate (effective 2026-05-20):** For
- projects created on or after 2026-05-20 on Langfuse Cloud,
- `LEGACY_TRACES_OBSERVATIONS` and
- `LEGACY_TRACES_AND_ENRICHED_OBSERVATIONS` are rejected with HTTP
- 400. Use `OBSERVATIONS_V2` for all new integrations. Self-hosted
- deployments are unaffected.
-
-
- **Cloud-only integration deprecation gate (effective 2026-06-22):**
- On Langfuse Cloud, legacy export sources are only accepted for blob
- storage integrations created before 2026-06-22, regardless of
- project age. Requests that would create a new integration with
- `LEGACY_TRACES_OBSERVATIONS` or
- `LEGACY_TRACES_AND_ENRICHED_OBSERVATIONS` are rejected with HTTP
- 400. Use `OBSERVATIONS_V2` instead. Self-hosted deployments are
- unaffected.
- exportFieldGroups:
- type: array
- items:
- $ref: '#/components/schemas/BlobStorageExportFieldGroup'
nullable: true
- description: >-
- Field groups to include in each exported observation row. Applies to
- all export sources; must include `core` if provided. When omitted on
- create, the column default (all groups) applies. When omitted on
- update, the existing value is preserved.
-
-
- `exportFieldGroups` requires `exportSource` to be provided in the
- same request.
+ description: The user ID of the comment author
required:
+ - id
- projectId
- - type
- - bucketName
- - region
- - exportFrequency
- - enabled
- - forcePathStyle
- - fileType
- - exportMode
- BlobStorageIntegrationResponse:
- title: BlobStorageIntegrationResponse
+ - createdAt
+ - updatedAt
+ - objectType
+ - objectId
+ - content
+ Dataset:
+ title: Dataset
type: object
properties:
id:
type: string
- projectId:
- type: string
- type:
- $ref: '#/components/schemas/BlobStorageIntegrationType'
- bucketName:
+ name:
type: string
- endpoint:
+ description:
type: string
nullable: true
- region:
- type: string
- accessKeyId:
- type: string
+ description: Description of the dataset
+ metadata:
+ description: Metadata associated with the dataset
+ inputSchema:
nullable: true
- prefix:
+ description: JSON Schema for validating dataset item inputs
+ expectedOutputSchema:
+ nullable: true
+ description: JSON Schema for validating dataset item expected outputs
+ projectId:
type: string
- exportFrequency:
- $ref: '#/components/schemas/BlobStorageExportFrequency'
- enabled:
- type: boolean
- forcePathStyle:
- type: boolean
- fileType:
- $ref: '#/components/schemas/BlobStorageIntegrationFileTypeResponse'
- exportMode:
- $ref: '#/components/schemas/BlobStorageExportMode'
- exportStartDate:
+ createdAt:
type: string
format: date-time
- nullable: true
- compressed:
- type: boolean
- exportSource:
- $ref: '#/components/schemas/BlobStorageExportSource'
- exportFieldGroups:
- type: array
- items:
- $ref: '#/components/schemas/BlobStorageExportFieldGroup'
- nullable: true
- description: >-
- Field groups included in each exported observation row. An empty
- list is treated as all groups during export.
- nextSyncAt:
+ updatedAt:
type: string
format: date-time
- nullable: true
- lastSyncAt:
+ required:
+ - id
+ - name
+ - description
+ - metadata
+ - inputSchema
+ - expectedOutputSchema
+ - projectId
+ - createdAt
+ - updatedAt
+ DatasetItem:
+ title: DatasetItem
+ type: object
+ properties:
+ id:
type: string
- format: date-time
- nullable: true
- lastError:
+ status:
+ $ref: '#/components/schemas/DatasetStatus'
+ input:
+ description: Input data for the dataset item
+ expectedOutput:
+ description: Expected output for the dataset item
+ metadata:
+ description: Metadata associated with the dataset item
+ sourceTraceId:
type: string
nullable: true
- lastErrorAt:
+ description: The trace ID that sourced this dataset item
+ sourceObservationId:
type: string
- format: date-time
nullable: true
+ description: The observation ID that sourced this dataset item
+ datasetId:
+ type: string
+ datasetName:
+ type: string
createdAt:
type: string
format: date-time
updatedAt:
type: string
format: date-time
+ mediaReferences:
+ type: array
+ items:
+ $ref: '#/components/schemas/DatasetItemMediaReference'
+ description: >-
+ Resolved Langfuse media references found in input, expectedOutput,
+ and metadata.
required:
- id
- - projectId
- - type
- - bucketName
- - endpoint
- - region
- - accessKeyId
- - prefix
- - exportFrequency
- - enabled
- - forcePathStyle
- - fileType
- - exportMode
- - exportStartDate
- - compressed
- - exportSource
- - exportFieldGroups
- - nextSyncAt
- - lastSyncAt
- - lastError
- - lastErrorAt
+ - status
+ - input
+ - expectedOutput
+ - metadata
+ - sourceTraceId
+ - sourceObservationId
+ - datasetId
+ - datasetName
- createdAt
- updatedAt
- BlobStorageIntegrationsResponse:
- title: BlobStorageIntegrationsResponse
+ - mediaReferences
+ DatasetItemMediaReference:
+ title: DatasetItemMediaReference
type: object
properties:
- data:
- type: array
- items:
- $ref: '#/components/schemas/BlobStorageIntegrationResponse'
+ field:
+ $ref: '#/components/schemas/DatasetItemMediaReferenceField'
+ description: The dataset item field containing the reference
+ referenceString:
+ type: string
+ description: >-
+ The Langfuse media reference string, e.g.
+ `@@@langfuseMedia:type=image/png|id=...|source=bytes@@@`
+ jsonPath:
+ type: string
+ description: >-
+ JSONPath of the string holding the reference within the field, e.g.
+ `$['image']`
+ media:
+ $ref: '#/components/schemas/DatasetItemMediaReferenceMedia'
+ description: The resolved media record.
required:
- - data
- BlobStorageSyncStatus:
- title: BlobStorageSyncStatus
+ - field
+ - referenceString
+ - jsonPath
+ - media
+ DatasetItemMediaReferenceField:
+ title: DatasetItemMediaReferenceField
type: string
enum:
- - idle
- - running
- - queued
- - up_to_date
- - disabled
- - error
- description: >-
- Sync status of the blob storage integration:
-
- - `disabled` — integration is not enabled
-
- - `error` — last export failed (see `lastError` for details)
-
- - `running` — an export job is currently being processed
-
- - `queued` — next export is overdue (`nextSyncAt` is in the past) and
- waiting to be picked up by the worker
-
- - `idle` — enabled but has never exported yet and no export is queued
-
- - `up_to_date` — all available data has been exported; next export is
- scheduled for the future
-
-
- **ETL usage**: poll this endpoint and check for `up_to_date` status.
- Compare `lastSyncAt` against your
-
- ETL bookmark to determine if new data is available. Note that exports
- run with a 20-minute lag buffer,
-
- so `lastSyncAt` will always be at least 20 minutes behind real-time.
- BlobStorageIntegrationStatusResponse:
- title: BlobStorageIntegrationStatusResponse
+ - input
+ - expectedOutput
+ - metadata
+ DatasetItemMediaReferenceMedia:
+ title: DatasetItemMediaReferenceMedia
+ type: object
+ properties:
+ mediaId:
+ type: string
+ description: The unique langfuse identifier of the media record
+ contentType:
+ type: string
+ description: The MIME type of the media record
+ contentLength:
+ type: integer
+ description: The size of the media record in bytes
+ url:
+ type: string
+ description: The signed download URL of the media record
+ urlExpiry:
+ type: string
+ description: The expiry date and time of the download URL
+ required:
+ - mediaId
+ - contentType
+ - contentLength
+ - url
+ - urlExpiry
+ DatasetRunItem:
+ title: DatasetRunItem
type: object
properties:
id:
type: string
- projectId:
+ datasetRunId:
type: string
- syncStatus:
- $ref: '#/components/schemas/BlobStorageSyncStatus'
- enabled:
- type: boolean
- lastSyncAt:
+ datasetRunName:
type: string
- format: date-time
- nullable: true
- description: >-
- End of the last successfully exported time window. Compare against
- your ETL bookmark to determine if new data is available. Null if the
- integration has never synced.
- nextSyncAt:
+ datasetItemId:
type: string
- format: date-time
- nullable: true
- description: When the next export is scheduled. Null if no sync has occurred yet.
- lastError:
+ traceId:
+ type: string
+ observationId:
type: string
nullable: true
- description: >-
- Raw error message from the storage provider (S3/Azure/GCS) if the
- last export failed. Cleared on successful export.
- lastErrorAt:
+ description: The observation ID associated with this run item
+ createdAt:
+ type: string
+ format: date-time
+ updatedAt:
type: string
format: date-time
- nullable: true
- description: When the last error occurred. Cleared on successful export.
required:
- id
- - projectId
- - syncStatus
- - enabled
- - lastSyncAt
- - nextSyncAt
- - lastError
- - lastErrorAt
- BlobStorageIntegrationDeletionResponse:
- title: BlobStorageIntegrationDeletionResponse
+ - datasetRunId
+ - datasetRunName
+ - datasetItemId
+ - traceId
+ - observationId
+ - createdAt
+ - updatedAt
+ DatasetRun:
+ title: DatasetRun
type: object
properties:
- message:
+ id:
type: string
- required:
- - message
- CreateCommentRequest:
- title: CreateCommentRequest
- type: object
- properties:
- projectId:
+ description: Unique identifier of the dataset run
+ name:
type: string
- description: The id of the project to attach the comment to.
- objectType:
+ description: Name of the dataset run
+ description:
type: string
- description: >-
- The type of the object to attach the comment to (trace, observation,
- session, prompt).
- objectId:
+ nullable: true
+ description: Description of the run
+ metadata:
+ description: Metadata of the dataset run
+ datasetId:
type: string
- description: >-
- The id of the object to attach the comment to. If this does not
- reference a valid existing object, an error will be thrown.
- content:
+ description: Id of the associated dataset
+ datasetName:
type: string
- description: >-
- The content of the comment. May include markdown. Currently limited
- to 5000 characters.
- authorUserId:
+ description: Name of the associated dataset
+ createdAt:
type: string
- nullable: true
- description: >-
- The id of the user who created the comment. Must be a member of the
- organization that owns the project, otherwise an error will be
- thrown.
- required:
- - projectId
- - objectType
- - objectId
- - content
- CreateCommentResponse:
- title: CreateCommentResponse
- type: object
- properties:
- id:
+ format: date-time
+ description: The date and time when the dataset run was created
+ updatedAt:
type: string
- description: The id of the created object in Langfuse
+ format: date-time
+ description: The date and time when the dataset run was last updated
required:
- id
- GetCommentsResponse:
- title: GetCommentsResponse
+ - name
+ - description
+ - metadata
+ - datasetId
+ - datasetName
+ - createdAt
+ - updatedAt
+ DatasetRunWithItems:
+ title: DatasetRunWithItems
type: object
properties:
- data:
+ datasetRunItems:
type: array
items:
- $ref: '#/components/schemas/Comment'
- meta:
- $ref: '#/components/schemas/utilsMetaResponse'
+ $ref: '#/components/schemas/DatasetRunItem'
+ _deprecation:
+ $ref: '#/components/schemas/Deprecation'
+ nullable: true
required:
- - data
- - meta
- Deprecation:
- title: Deprecation
+ - datasetRunItems
+ allOf:
+ - $ref: '#/components/schemas/DatasetRun'
+ Model:
+ title: Model
type: object
description: >-
- Migration signal returned by deprecated endpoints. Optional fields are
- omitted when they have no value.
+ Model definition used for transforming usage into USD cost and/or
+ tokenization.
+
+
+ Models can have either simple flat pricing or tiered pricing:
+
+ - Flat pricing: Single price per usage type (legacy, but still
+ supported)
+
+ - Tiered pricing: Multiple pricing tiers with conditional matching based
+ on usage patterns or observation attributes
+
+
+ The pricing tiers approach is recommended for models with usage-based
+ pricing variations.
+
+ When using tiered pricing, the flat price fields (inputPrice,
+ outputPrice, prices) are populated
+
+ from the default tier for backward compatibility.
properties:
- message:
+ id:
type: string
- description: >-
- Human- and agent-readable summary of the deprecation and its
- replacement.
- replacement:
+ modelName:
type: string
- nullable: true
description: >-
- The replacement endpoint, e.g. "GET /api/public/v2/observations".
- Omitted when the endpoint is being removed without a direct
- replacement.
- docsUrl:
- type: string
- nullable: true
- description: Link to the migration documentation (markdown), when available.
- sunsetAt:
+ Name of the model definition. If multiple with the same name exist,
+ they are applied in the following order: (1) custom over built-in,
+ (2) newest according to startTime where
+ model.startTime<observation.startTime
+ matchPattern:
type: string
- nullable: true
description: >-
- ISO date after which the endpoint may stop working, when a removal
- date is committed.
- required:
- - message
- Trace:
- title: Trace
- type: object
- properties:
- id:
- type: string
- description: The unique identifier of a trace
- timestamp:
+ Regex pattern which matches this model definition to
+ generation.model. Useful in case of fine-tuned models. If you want
+ to exact match, use `(?i)^modelname$`
+ startDate:
type: string
format: date-time
- description: The timestamp when the trace was created
- name:
- type: string
nullable: true
- description: The name of the trace
- input:
+ description: Apply only to generations which are newer than this ISO date.
+ unit:
+ $ref: '#/components/schemas/ModelUsageUnit'
nullable: true
- description: The input data of the trace. Can be any JSON.
- output:
+ description: Unit used by this model.
+ inputPrice:
+ type: number
+ format: double
nullable: true
- description: The output data of the trace. Can be any JSON.
- sessionId:
- type: string
+ description: Deprecated. See 'prices' instead. Price (USD) per input unit
+ outputPrice:
+ type: number
+ format: double
nullable: true
- description: The session identifier associated with the trace
- release:
- type: string
+ description: Deprecated. See 'prices' instead. Price (USD) per output unit
+ totalPrice:
+ type: number
+ format: double
nullable: true
- description: The release version of the application when the trace was created
- version:
+ description: >-
+ Deprecated. See 'prices' instead. Price (USD) per total unit. Cannot
+ be set if input or output price is set.
+ tokenizerId:
type: string
nullable: true
- description: The version of the trace
- userId:
+ description: >-
+ Optional. Tokenizer to be applied to observations which match to
+ this model. See docs for more details.
+ tokenizerConfig:
+ description: >-
+ Optional. Configuration for the selected tokenizer. Needs to be
+ JSON. See docs for more details.
+ isLangfuseManaged:
+ type: boolean
+ createdAt:
type: string
- nullable: true
- description: The user identifier associated with the trace
- metadata:
- nullable: true
- description: The metadata associated with the trace. Can be any JSON.
- tags:
+ format: date-time
+ description: Timestamp when the model was created
+ prices:
+ type: object
+ additionalProperties:
+ $ref: '#/components/schemas/ModelPrice'
+ description: >-
+ Deprecated. Use 'pricingTiers' instead for models with usage-based
+ pricing variations.
+
+
+ This field shows prices by usage type from the default pricing tier.
+ Maintained for backward compatibility.
+
+ If the model uses tiered pricing, this field will be populated from
+ the default tier's prices.
+ pricingTiers:
type: array
items:
- type: string
- description: The tags associated with the trace.
- public:
- type: boolean
- description: Public traces are accessible via url without login
- environment:
- type: string
+ $ref: '#/components/schemas/PricingTier'
description: >-
- The environment from which this trace originated. Can be any
- lowercase alphanumeric string with hyphens and underscores that does
- not start with 'langfuse'.
+ Array of pricing tiers with conditional pricing based on usage
+ thresholds or observation attributes.
+
+
+ Pricing tiers enable accurate cost tracking for models that charge
+ different rates based on usage patterns
+
+ or request attributes (e.g., high-volume usage, large context
+ windows, cached tokens, or service tiers).
+
+
+ Each model must have exactly one default tier (isDefault=true,
+ priority=0) that serves as a fallback.
+
+ Additional conditional tiers can be defined with specific matching
+ criteria.
+
+
+ If this array is empty, the model uses legacy flat pricing from the
+ inputPrice/outputPrice/totalPrice fields.
required:
- id
- - timestamp
- - name
- - sessionId
- - release
- - version
- - userId
- - tags
- - public
- - environment
- TraceWithDetails:
- title: TraceWithDetails
+ - modelName
+ - matchPattern
+ - startDate
+ - inputPrice
+ - outputPrice
+ - totalPrice
+ - tokenizerId
+ - tokenizerConfig
+ - isLangfuseManaged
+ - createdAt
+ - prices
+ - pricingTiers
+ ModelPrice:
+ title: ModelPrice
+ type: object
+ properties:
+ price:
+ type: number
+ format: double
+ required:
+ - price
+ PricingTierCondition:
+ title: PricingTierCondition
+ oneOf:
+ - $ref: '#/components/schemas/PricingTierUsageCondition'
+ - $ref: '#/components/schemas/PricingTierAttributeCondition'
+ description: >-
+ Condition for matching a pricing tier against usage details or
+ observation attributes.
+
+
+ Usage-detail conditions treat usageDetailPattern as a regex, sum all
+ matching usage values, and compare the sum to the numeric value.
+ Model-parameter and metadata conditions match an exact top-level key
+ against one or more string values.
+ PricingTierUsageCondition:
+ title: PricingTierUsageCondition
type: object
+ description: Condition that sums usage details whose keys match a regex.
properties:
- htmlPath:
+ usageDetailPattern:
type: string
- description: Path of trace in Langfuse UI
- latency:
- type: number
- format: double
- nullable: true
- description: Latency of trace in seconds
- totalCost:
+ description: Regex pattern matched against usage detail keys.
+ operator:
+ $ref: '#/components/schemas/PricingTierOperator'
+ value:
type: number
format: double
- nullable: true
- description: Cost of trace in USD
- observations:
- type: array
- items:
- type: string
- nullable: true
- description: List of observation ids
- scores:
- type: array
- items:
- type: string
- nullable: true
- description: List of score ids
+ description: Numeric threshold for the summed matching usage values.
+ caseSensitive:
+ type: boolean
+ description: Whether the usage-detail regex is case-sensitive.
required:
- - htmlPath
- allOf:
- - $ref: '#/components/schemas/Trace'
- TraceWithFullDetails:
- title: TraceWithFullDetails
+ - usageDetailPattern
+ - operator
+ - value
+ - caseSensitive
+ PricingTierAttributeCondition:
+ title: PricingTierAttributeCondition
type: object
+ description: >-
+ Condition that matches any configured value for a top-level observation
+ attribute.
properties:
- htmlPath:
+ source:
+ $ref: '#/components/schemas/PricingTierAttributeSource'
+ description: Observation attribute object evaluated by this condition.
+ key:
type: string
- description: Path of trace in Langfuse UI
- latency:
- type: number
- format: double
- nullable: true
- description: Latency of trace in seconds
- totalCost:
- type: number
- format: double
- nullable: true
- description: Cost of trace in USD
- observations:
- type: array
- items:
- $ref: '#/components/schemas/ObservationsView'
- description: List of observations
- scores:
+ description: Exact top-level attribute key.
+ operator:
+ type: string
+ const: in
+ description: Membership operator.
+ values:
type: array
items:
- $ref: '#/components/schemas/ScoreV1'
- description: List of scores
- _deprecation:
- $ref: '#/components/schemas/Deprecation'
- nullable: true
+ type: string
+ description: Accepted string attribute values. At least one value is required.
required:
- - htmlPath
- - observations
- - scores
- allOf:
- - $ref: '#/components/schemas/Trace'
- Session:
- title: Session
+ - source
+ - key
+ - operator
+ - values
+ PricingTierAttributeSource:
+ title: PricingTierAttributeSource
+ type: string
+ enum:
+ - model_parameters
+ - metadata
+ PricingTierConditionInput:
+ title: PricingTierConditionInput
+ oneOf:
+ - $ref: '#/components/schemas/PricingTierUsageConditionInput'
+ - $ref: '#/components/schemas/PricingTierAttributeCondition'
+ description: Condition input for matching a pricing tier.
+ PricingTierUsageConditionInput:
+ title: PricingTierUsageConditionInput
type: object
+ description: Input condition that sums usage details whose keys match a regex.
properties:
- id:
- type: string
- createdAt:
- type: string
- format: date-time
- projectId:
- type: string
- environment:
+ usageDetailPattern:
type: string
- description: The environment from which this session originated.
- required:
- - id
- - createdAt
- - projectId
- - environment
- SessionWithTraces:
- title: SessionWithTraces
- type: object
- properties:
- traces:
- type: array
- items:
- $ref: '#/components/schemas/Trace'
- _deprecation:
- $ref: '#/components/schemas/Deprecation'
+ description: Regex pattern matched against usage detail keys.
+ operator:
+ $ref: '#/components/schemas/PricingTierOperator'
+ value:
+ type: number
+ format: double
+ description: Numeric threshold for the summed matching usage values.
+ caseSensitive:
+ type: boolean
nullable: true
+ description: Whether the usage-detail regex is case-sensitive. Defaults to false.
required:
- - traces
- allOf:
- - $ref: '#/components/schemas/Session'
- Observation:
- title: Observation
+ - usageDetailPattern
+ - operator
+ - value
+ PricingTier:
+ title: PricingTier
type: object
+ description: >-
+ Pricing tier definition with conditional pricing based on usage
+ thresholds or observation attributes.
+
+
+ Pricing tiers enable accurate cost tracking for LLM providers that
+ charge different rates based on usage patterns or request attributes.
+
+ For example, some providers charge higher rates when context size
+ exceeds certain thresholds or when an accelerated service tier is
+ selected.
+
+
+ How tier matching works:
+
+ 1. Tiers are evaluated in ascending priority order (priority 1 before
+ priority 2, etc.)
+
+ 2. The first tier where ALL conditions match is selected
+
+ 3. If no conditional tiers match, the default tier is used as a fallback
+
+ 4. The default tier has priority 0 and no conditions
+
+
+ Why priorities matter:
+
+ - Lower priority numbers are evaluated first, allowing you to define
+ specific cases before general ones
+
+ - Example: Priority 1 for "high usage" (>200K tokens), Priority 2 for
+ "medium usage" (>100K tokens), Priority 0 for default
+
+ - Without proper ordering, a less specific condition might match before
+ a more specific one
+
+
+ Every model must have exactly one default tier to ensure cost
+ calculation always succeeds.
properties:
id:
type: string
- description: The unique identifier of the observation
- traceId:
- type: string
- nullable: true
- description: The trace ID associated with the observation
- type:
- type: string
- description: The type of the observation
+ description: Unique identifier for the pricing tier
name:
type: string
- nullable: true
- description: The name of the observation
- startTime:
- type: string
- format: date-time
- description: The start time of the observation
- endTime:
- type: string
- format: date-time
- nullable: true
- description: The end time of the observation.
- completionStartTime:
- type: string
- format: date-time
- nullable: true
- description: The completion start time of the observation
- model:
- type: string
- nullable: true
- description: The model used for the observation
- modelParameters:
- description: The parameters of the model used for the observation
- input:
- description: The input data of the observation
- version:
- type: string
- nullable: true
- description: The version of the observation
- metadata:
- description: Additional metadata of the observation
- output:
- description: The output data of the observation
- usage:
- $ref: '#/components/schemas/Usage'
description: >-
- (Deprecated. Use usageDetails and costDetails instead.) The usage
- data of the observation
- level:
- $ref: '#/components/schemas/ObservationLevel'
- description: The level of the observation
- statusMessage:
- type: string
- nullable: true
- description: The status message of the observation
- parentObservationId:
- type: string
- nullable: true
- description: The parent observation ID
- promptId:
- type: string
- nullable: true
- description: The prompt ID associated with the observation
- usageDetails:
- type: object
- additionalProperties:
- type: integer
+ Name of the pricing tier for display and identification purposes.
+
+
+ Examples: "Standard", "High Volume Tier", "Large Context", "Extended
+ Context Tier"
+ isDefault:
+ type: boolean
+ description: >-
+ Whether this is the default tier. Every model must have exactly one
+ default tier with priority 0 and no conditions.
+
+
+ The default tier serves as a fallback when no conditional tiers
+ match, ensuring cost calculation always succeeds.
+
+ It typically represents the base pricing for standard usage
+ patterns.
+ priority:
+ type: integer
+ description: >-
+ Priority for tier matching evaluation. Lower numbers = higher
+ priority (evaluated first).
+
+
+ The default tier must always have priority 0. Conditional tiers
+ should have priority 1, 2, 3, etc.
+
+
+ Example ordering:
+
+ - Priority 0: Default tier (no conditions, always matches as
+ fallback)
+
+ - Priority 1: High usage tier (e.g., >200K tokens)
+
+ - Priority 2: Medium usage tier (e.g., >100K tokens)
+
+
+ This ensures more specific conditions are checked before general
+ ones.
+ conditions:
+ type: array
+ items:
+ $ref: '#/components/schemas/PricingTierCondition'
description: >-
- The usage details of the observation. Key is the name of the usage
- metric, value is the number of units consumed. The total key is the
- sum of all (non-total) usage metrics or the total value ingested.
- costDetails:
+ Array of conditions that must ALL be met for this tier to match (AND
+ logic).
+
+
+ The default tier must have an empty conditions array. Conditional
+ tiers should have one or more conditions
+
+ that define when this tier's pricing applies.
+
+
+ Multiple conditions enable complex matching scenarios (e.g., "high
+ input tokens AND low output tokens").
+ prices:
type: object
additionalProperties:
type: number
format: double
description: >-
- The cost details of the observation. Key is the name of the cost
- metric, value is the cost in USD. The total key is the sum of all
- (non-total) cost metrics or the total value ingested.
- environment:
- type: string
- description: >-
- The environment from which this observation originated. Can be any
- lowercase alphanumeric string with hyphens and underscores that does
- not start with 'langfuse'.
+ Prices (USD) by usage type for this tier.
+
+
+ Common usage types: "input", "output", "total", "request", "image"
+
+ Prices are specified in USD per unit (e.g., per token, per request,
+ per second).
+
+
+ Example: {"input": 0.000003, "output": 0.000015} means $3 per
+ million input tokens and $15 per million output tokens.
required:
- id
- - traceId
- - type
- name
- - startTime
- - endTime
- - completionStartTime
- - model
- - modelParameters
- - input
- - version
- - metadata
- - output
- - usage
- - level
- - statusMessage
- - parentObservationId
- - promptId
- - usageDetails
- - costDetails
- - environment
- ObservationsView:
- title: ObservationsView
+ - isDefault
+ - priority
+ - conditions
+ - prices
+ PricingTierInput:
+ title: PricingTierInput
type: object
+ description: >-
+ Input schema for creating a pricing tier. The tier ID will be
+ automatically generated server-side.
+
+
+ When creating a model with pricing tiers:
+
+ - Exactly one tier must have isDefault=true (the fallback tier)
+
+ - The default tier must have priority=0 and conditions=[]
+
+ - All tier names and priorities must be unique within the model
+
+ - Each tier must define at least one price
+
+
+ See PricingTier for detailed information about how tiers work and why
+ they're useful.
properties:
- promptName:
- type: string
- nullable: true
- description: The name of the prompt associated with the observation
- promptVersion:
- type: integer
- nullable: true
- description: The version of the prompt associated with the observation
- modelId:
+ name:
type: string
- nullable: true
- description: The unique identifier of the model
- inputPrice:
- type: number
- format: double
- nullable: true
- description: The price of the input in USD
- outputPrice:
- type: number
- format: double
- nullable: true
- description: The price of the output in USD.
- totalPrice:
- type: number
- format: double
- nullable: true
- description: The total price in USD.
- calculatedInputCost:
- type: number
- format: double
- nullable: true
description: >-
- (Deprecated. Use usageDetails and costDetails instead.) The
- calculated cost of the input in USD
- calculatedOutputCost:
- type: number
- format: double
- nullable: true
- description: >-
- (Deprecated. Use usageDetails and costDetails instead.) The
- calculated cost of the output in USD
- calculatedTotalCost:
- type: number
- format: double
+ Name of the pricing tier for display and identification purposes.
+
+
+ Must be unique within the model. Common patterns: "Standard", "High
+ Volume Tier", "Extended Context"
+ isDefault:
+ type: boolean
nullable: true
description: >-
- (Deprecated. Use usageDetails and costDetails instead.) The
- calculated total cost in USD
- latency:
- type: number
- format: double
- nullable: true
- description: The latency in seconds.
- timeToFirstToken:
- type: number
- format: double
- nullable: true
- description: The time to the first token in seconds
- required:
- - promptName
- - promptVersion
- - modelId
- - inputPrice
- - outputPrice
- - totalPrice
- - calculatedInputCost
- - calculatedOutputCost
- - calculatedTotalCost
- - latency
- - timeToFirstToken
- allOf:
- - $ref: '#/components/schemas/Observation'
- ObservationsViewSingle:
- title: ObservationsViewSingle
- type: object
- properties:
- _deprecation:
- $ref: '#/components/schemas/Deprecation'
- nullable: true
- allOf:
- - $ref: '#/components/schemas/ObservationsView'
- ObservationV2:
- title: ObservationV2
- type: object
- description: >-
- An observation from the v2 API with field-group-based selection.
+ Whether this is the default tier. Exactly one tier per model must be
+ marked as default.
- Core fields are always present. Other fields are included only when
- their field group is requested.
- properties:
- id:
- type: string
- description: The unique identifier of the observation
- traceId:
- type: string
- nullable: true
- description: The trace ID associated with the observation
- startTime:
- type: string
- format: date-time
- description: The start time of the observation
- endTime:
- type: string
- format: date-time
- nullable: true
- description: The end time of the observation
- projectId:
- type: string
- description: The project ID this observation belongs to
- parentObservationId:
- type: string
- nullable: true
+
+ Requirements for default tier:
+
+ - Must have isDefault=true
+
+ - Must have priority=0
+
+ - Must have empty conditions array (conditions=[])
+
+
+ The default tier acts as a fallback when no conditional tiers match.
+ Defaults to false when omitted.
+ priority:
+ type: integer
description: >-
- The physical parent observation ID, if present.
+ Priority for tier matching evaluation. Lower numbers = higher
+ priority (evaluated first).
- Observations marked as app roots by the SDK may retain a non-null
- parent ID.
- type:
- type: string
- description: The type of the observation (e.g. GENERATION, SPAN, EVENT)
- isRootObservation:
- type: boolean
- nullable: true
+
+ Must be unique within the model. The default tier must have
+ priority=0.
+
+ Conditional tiers should use priority 1, 2, 3, etc. based on their
+ specificity.
+ conditions:
+ type: array
+ items:
+ $ref: '#/components/schemas/PricingTierConditionInput'
description: >-
- Whether this observation is a logical root.
+ Array of conditions that must ALL be met for this tier to match (AND
+ logic).
- This is true for observations without a physical parent and
- observations marked as app roots by the SDK.
- name:
- type: string
- nullable: true
- description: The name of the observation
- level:
- $ref: '#/components/schemas/ObservationLevel'
- nullable: true
- description: The level of the observation
- statusMessage:
- type: string
- nullable: true
- description: The status message of the observation
- version:
- type: string
- nullable: true
- description: The version of the observation
- environment:
- type: string
- nullable: true
- description: The environment from which this observation originated
- bookmarked:
- type: boolean
+
+ The default tier must have an empty array (conditions=[]).
+
+ Conditional tiers should define one or more conditions that specify
+ when this tier's pricing applies.
+
+
+ Conditions can compare summed matching usage details to a numeric
+ threshold, or exactly match a top-level model parameter or metadata
+ value.
+ prices:
+ type: object
+ additionalProperties:
+ type: number
+ format: double
+ description: >-
+ Prices (USD) by usage type for this tier. At least one price must be
+ defined.
+
+
+ Common usage types: "input", "output", "total", "request", "image"
+
+ Prices are in USD per unit (e.g., per token).
+
+
+ Example: {"input": 0.000003, "output": 0.000015} represents $3 per
+ million input tokens and $15 per million output tokens.
+ required:
+ - name
+ - priority
+ - conditions
+ - prices
+ PricingTierOperator:
+ title: PricingTierOperator
+ type: string
+ enum:
+ - gt
+ - gte
+ - lt
+ - lte
+ - eq
+ - neq
+ description: Comparison operators for pricing tier conditions
+ ModelUsageUnit:
+ title: ModelUsageUnit
+ type: string
+ enum:
+ - CHARACTERS
+ - TOKENS
+ - MILLISECONDS
+ - SECONDS
+ - IMAGES
+ - REQUESTS
+ description: Unit of usage in Langfuse
+ ObservationLevel:
+ title: ObservationLevel
+ type: string
+ enum:
+ - DEBUG
+ - DEFAULT
+ - WARNING
+ - ERROR
+ MapValue:
+ title: MapValue
+ oneOf:
+ - type: string
nullable: true
- description: Whether the observation is bookmarked
- public:
- type: boolean
+ - type: integer
nullable: true
- description: Whether the observation is public
- userId:
- type: string
+ - type: number
+ format: float
nullable: true
- description: The user ID associated with the observation
- sessionId:
- type: string
+ - type: boolean
nullable: true
- description: The session ID associated with the observation
- completionStartTime:
- type: string
- format: date-time
+ - type: array
+ items:
+ type: string
nullable: true
- description: The completion start time of the observation
- createdAt:
+ CommentObjectType:
+ title: CommentObjectType
+ type: string
+ enum:
+ - TRACE
+ - OBSERVATION
+ - SESSION
+ - PROMPT
+ DatasetStatus:
+ title: DatasetStatus
+ type: string
+ enum:
+ - ACTIVE
+ - ARCHIVED
+ ScoreSource:
+ title: ScoreSource
+ type: string
+ enum:
+ - ANNOTATION
+ - API
+ - EVAL
+ ScoreConfigDataType:
+ title: ScoreConfigDataType
+ type: string
+ enum:
+ - NUMERIC
+ - BOOLEAN
+ - CATEGORICAL
+ - TEXT
+ ScoreDataType:
+ title: ScoreDataType
+ type: string
+ enum:
+ - NUMERIC
+ - BOOLEAN
+ - CATEGORICAL
+ - CORRECTION
+ - TEXT
+ DeleteDatasetItemResponse:
+ title: DeleteDatasetItemResponse
+ type: object
+ properties:
+ message:
type: string
- format: date-time
- nullable: true
- description: The creation timestamp of the observation
- updatedAt:
+ description: Success message after deletion
+ required:
+ - message
+ CreateDatasetItemRequest:
+ title: CreateDatasetItemRequest
+ type: object
+ properties:
+ datasetName:
type: string
- format: date-time
- nullable: true
- description: The last update timestamp of the observation
input:
nullable: true
- description: The input data of the observation
- output:
+ expectedOutput:
nullable: true
- description: The output data of the observation
metadata:
nullable: true
- description: Additional metadata of the observation
- providedModelName:
+ sourceTraceId:
type: string
nullable: true
- description: The model name as provided by the user
- internalModelId:
+ sourceObservationId:
type: string
nullable: true
- description: The internal model ID matched by Langfuse
- modelParameters:
- nullable: true
- description: The parameters of the model used for the observation
- usageDetails:
- type: object
- additionalProperties:
- type: integer
- nullable: true
- description: >-
- The usage details of the observation. Key is the usage metric name,
- value is the number of units consumed.
- costDetails:
- type: object
- additionalProperties:
- type: number
- format: double
- nullable: true
- description: >-
- The cost details of the observation. Key is the cost metric name,
- value is the cost in USD.
- totalCost:
- type: number
- format: double
- nullable: true
- description: The total cost of the observation in USD
- usagePricingTierName:
+ id:
type: string
nullable: true
description: >-
- The name of the pricing tier applied to this observation's usage
- costs
- promptId:
+ Dataset items are upserted on their id. Id needs to be unique
+ (project-level), cannot be reused across datasets, and must be at
+ most 255 characters.
+ status:
+ $ref: '#/components/schemas/DatasetStatus'
+ nullable: true
+ description: Defaults to ACTIVE for newly created items
+ required:
+ - datasetName
+ PaginatedDatasetItems:
+ title: PaginatedDatasetItems
+ type: object
+ properties:
+ data:
+ type: array
+ items:
+ $ref: '#/components/schemas/DatasetItem'
+ meta:
+ $ref: '#/components/schemas/utilsMetaResponse'
+ required:
+ - data
+ - meta
+ CreateDatasetRunItemRequest:
+ title: CreateDatasetRunItemRequest
+ type: object
+ properties:
+ runName:
type: string
- nullable: true
- description: The prompt ID associated with the observation
- promptName:
+ runDescription:
type: string
nullable: true
- description: The prompt name associated with the observation
- promptVersion:
- type: integer
- nullable: true
- description: The prompt version associated with the observation
- latency:
- type: number
- format: double
+ description: Description of the run. If run exists, description will be updated.
+ metadata:
nullable: true
- description: The latency in seconds
- timeToFirstToken:
- type: number
- format: double
+ description: Metadata of the dataset run, updates run if run already exists
+ datasetItemId:
+ type: string
+ observationId:
+ type: string
nullable: true
- description: The time to first token in seconds
- modelId:
+ traceId:
type: string
nullable: true
description: >-
- The matched model ID. Null when the `model` field group is not
- requested.
- inputPrice:
+ traceId should always be provided. For compatibility with older SDK
+ versions it can also be inferred from the provided observationId.
+ datasetVersion:
type: string
+ format: date-time
nullable: true
description: >-
- The input token price (USD per unit) from the matched model,
- serialized as a decimal string (e.g. "0.0001"). Null when the
- `model` field group is not requested.
- outputPrice:
+ ISO 8601 timestamp (RFC 3339, Section 5.6) in UTC (e.g.,
+ "2026-01-21T14:35:42Z").
+
+ Specifies the dataset version to use for this experiment run.
+
+ If provided, the experiment will use dataset items as they existed
+ at or before this timestamp.
+
+ If not provided, uses the latest version of dataset items.
+ createdAt:
type: string
+ format: date-time
nullable: true
description: >-
- The output token price (USD per unit) from the matched model,
- serialized as a decimal string (e.g. "0.0001"). Null when the
- `model` field group is not requested.
- totalPrice:
+ Optional timestamp to set the createdAt field of the dataset run
+ item. If not provided or null, defaults to current timestamp.
+ required:
+ - runName
+ - datasetItemId
+ PaginatedDatasetRunItems:
+ title: PaginatedDatasetRunItems
+ type: object
+ properties:
+ data:
+ type: array
+ items:
+ $ref: '#/components/schemas/DatasetRunItem'
+ meta:
+ $ref: '#/components/schemas/utilsMetaResponse'
+ _deprecation:
+ $ref: '#/components/schemas/Deprecation'
+ nullable: true
+ required:
+ - data
+ - meta
+ PaginatedDatasets:
+ title: PaginatedDatasets
+ type: object
+ properties:
+ data:
+ type: array
+ items:
+ $ref: '#/components/schemas/Dataset'
+ meta:
+ $ref: '#/components/schemas/utilsMetaResponse'
+ required:
+ - data
+ - meta
+ CreateDatasetRequest:
+ title: CreateDatasetRequest
+ type: object
+ properties:
+ name:
+ type: string
+ description:
type: string
nullable: true
+ metadata:
+ nullable: true
+ inputSchema:
+ nullable: true
description: >-
- The total token price (USD per unit) from the matched model,
- serialized as a decimal string (e.g. "0.0001"). Null when the
- `model` field group is not requested.
- traceName:
- type: string
+ JSON Schema for validating dataset item inputs. When set, all new
+ and existing dataset items will be validated against this schema.
+ expectedOutputSchema:
nullable: true
- description: The name of the parent trace
- tags:
+ description: >-
+ JSON Schema for validating dataset item expected outputs. When set,
+ all new and existing dataset items will be validated against this
+ schema.
+ required:
+ - name
+ PaginatedDatasetRuns:
+ title: PaginatedDatasetRuns
+ type: object
+ properties:
+ data:
type: array
items:
- type: string
- nullable: true
- description: Tags from the parent trace (denormalized onto the observation)
- release:
- type: string
+ $ref: '#/components/schemas/DatasetRun'
+ meta:
+ $ref: '#/components/schemas/utilsMetaResponse'
+ _deprecation:
+ $ref: '#/components/schemas/Deprecation'
nullable: true
- description: The release version of the parent trace
required:
- - id
- - traceId
- - startTime
- - endTime
- - projectId
- - parentObservationId
- - type
- - modelId
- - inputPrice
- - outputPrice
- - totalPrice
- Usage:
- title: Usage
+ - data
+ - meta
+ DeleteDatasetRunResponse:
+ title: DeleteDatasetRunResponse
type: object
+ properties:
+ message:
+ type: string
+ required:
+ - message
+ EvaluatorType:
+ title: EvaluatorType
+ type: string
+ enum:
+ - llm_as_judge
+ - code
+ description: |-
+ The evaluator type.
+
+ The public API supports LLM-as-a-judge and code evaluators.
+ CodeEvaluatorSourceCodeLanguage:
+ title: CodeEvaluatorSourceCodeLanguage
+ type: string
+ enum:
+ - PYTHON
+ - TYPESCRIPT
+ description: Code evaluator runtime language.
+ PromptVariableMappingSource:
+ title: PromptVariableMappingSource
+ type: string
+ enum:
+ - input
+ - output
+ - metadata
+ - tool_calls
+ - expected_output
+ - experiment_item_metadata
description: >-
- (Deprecated. Use usageDetails and costDetails instead.) Standard
- interface for usage and cost
+ Source field used to populate a prompt variable.
+
+
+ Use these values when mapping evaluator prompt variables to live data.
+
+
+ Source semantics:
+
+ - `input`: the observation input payload
+
+ - `output`: the observation output payload
+
+ - `metadata`: the observation metadata object. Combine with `jsonPath`
+ when you need one nested field instead of the whole object.
+
+ - `tool_calls`: the tool calls recorded on the observation, as an array
+ of `{id, name, arguments, type, index}` objects in the order the model
+ emitted them. Combine with `jsonPath` (for example `$[*].name`) to
+ select parts of each call.
+
+ - `expected_output`: the experiment item's expected output when the
+ observation belongs to an experiment.
+
+ - `experiment_item_metadata`: the experiment item's metadata when the
+ observation belongs to an experiment.
+ EvaluatorOutputScoreType:
+ title: EvaluatorOutputScoreType
+ type: string
+ enum:
+ - NUMERIC
+ - BOOLEAN
+ - CATEGORICAL
+ description: >-
+ Structured score type returned by an evaluator.
+
+
+ This controls the type of score value Langfuse stores for evaluation
+ results:
+
+ - `NUMERIC`: a numeric score such as `0.82`
+
+ - `BOOLEAN`: a boolean score such as `true`
+
+ - `CATEGORICAL`: one or more category labels from a fixed list
+ EvaluatorOutputDefinition:
+ title: EvaluatorOutputDefinition
+ oneOf:
+ - type: object
+ allOf:
+ - type: object
+ properties:
+ dataType:
+ type: string
+ enum:
+ - NUMERIC
+ - $ref: '#/components/schemas/PublicEvaluatorNumericScore'
+ required:
+ - dataType
+ - type: object
+ allOf:
+ - type: object
+ properties:
+ dataType:
+ type: string
+ enum:
+ - BOOLEAN
+ - $ref: '#/components/schemas/PublicEvaluatorBooleanScore'
+ required:
+ - dataType
+ - type: object
+ allOf:
+ - type: object
+ properties:
+ dataType:
+ type: string
+ enum:
+ - CATEGORICAL
+ - $ref: '#/components/schemas/PublicEvaluatorCategoricalScore'
+ required:
+ - dataType
+ description: >-
+ Flat structured output definition used when creating or updating an
+ evaluator.
+
+
+ - `dataType` is required.
+
+ - `scoreReasoningInstructions` and `scoreValueInstructions` are optional
+ instructions.
+
+ - `minValue` and `maxValue` apply only to `NUMERIC` outputs. If both are
+ set, `minValue` must not exceed `maxValue`.
+
+ - `categories` and `shouldAllowMultipleMatches` apply only to
+ `CATEGORICAL` outputs.
+
+ - Do not send `version`; that is an internal storage detail.
+ EvaluatorOutputDefinitionBase:
+ title: EvaluatorOutputDefinitionBase
+ type: object
properties:
- input:
- type: integer
- description: Number of input units (e.g. tokens)
- output:
- type: integer
- description: Number of output units (e.g. tokens)
- total:
- type: integer
- description: Defaults to input+output if not set
- unit:
+ scoreReasoningInstructions:
type: string
nullable: true
- description: Unit of measurement
- inputCost:
- type: number
- format: double
- nullable: true
- description: USD input cost
- outputCost:
- type: number
- format: double
- nullable: true
- description: USD output cost
- totalCost:
- type: number
- format: double
+ description: >-
+ Optional instructions for deriving the reasoning returned with the
+ score.
+ scoreValueInstructions:
+ type: string
nullable: true
- description: USD total cost, defaults to input+output
- required:
- - input
- - output
- - total
- - unit
- ScoreConfig:
- title: ScoreConfig
+ description: Optional instructions for deriving the score value.
+ PublicEvaluatorNumericScore:
+ title: PublicEvaluatorNumericScore
type: object
- description: Configuration for a score
properties:
- id:
- type: string
- name:
- type: string
- createdAt:
- type: string
- format: date-time
- updatedAt:
- type: string
- format: date-time
- projectId:
- type: string
dataType:
- $ref: '#/components/schemas/ScoreConfigDataType'
- isArchived:
- type: boolean
- description: Whether the score config is archived. Defaults to false
+ type: string
+ const: NUMERIC
+ description: Numeric score output.
minValue:
type: number
format: double
nullable: true
- description: >-
- Sets minimum value for numerical scores. If not set, the minimum
- value defaults to -∞
+ description: Optional inclusive minimum value.
maxValue:
type: number
format: double
nullable: true
- description: >-
- Sets maximum value for numerical scores. If not set, the maximum
- value defaults to +∞
- categories:
- type: array
- items:
- $ref: '#/components/schemas/ConfigCategory'
- nullable: true
- description: Configures custom categories for categorical scores
- description:
- type: string
- nullable: true
- description: Description of the score config
+ description: Optional inclusive maximum value.
required:
- - id
- - name
- - createdAt
- - updatedAt
- - projectId
- dataType
- - isArchived
- ConfigCategory:
- title: ConfigCategory
- type: object
- properties:
- value:
- type: number
- format: double
- label:
- type: string
- required:
- - value
- - label
- BaseScoreV1:
- title: BaseScoreV1
- type: object
- properties:
- id:
- type: string
- traceId:
- type: string
- name:
- type: string
- source:
- $ref: '#/components/schemas/ScoreSource'
- observationId:
- type: string
- nullable: true
- description: The observation ID associated with the score
- timestamp:
- type: string
- format: date-time
- createdAt:
- type: string
- format: date-time
- updatedAt:
- type: string
- format: date-time
- authorUserId:
- type: string
- nullable: true
- description: The user ID of the author
- comment:
- type: string
- nullable: true
- description: Comment on the score
- metadata:
- description: Metadata associated with the score
- configId:
- type: string
- nullable: true
- description: >-
- Reference a score config on a score. When set, config and score name
- must be equal and value must comply to optionally defined numerical
- range
- queueId:
- type: string
- nullable: true
- description: >-
- The annotation queue referenced by the score. Indicates if score was
- initially created while processing annotation queue.
- environment:
- type: string
- description: >-
- The environment from which this score originated. Can be any
- lowercase alphanumeric string with hyphens and underscores that does
- not start with 'langfuse'.
- required:
- - id
- - traceId
- - name
- - source
- - timestamp
- - createdAt
- - updatedAt
- - authorUserId
- - comment
- - metadata
- - configId
- - queueId
- - environment
- NumericScoreV1:
- title: NumericScoreV1
- type: object
- properties:
- value:
- type: number
- format: double
- description: The numeric value of the score
- required:
- - value
- allOf:
- - $ref: '#/components/schemas/BaseScoreV1'
- BooleanScoreV1:
- title: BooleanScoreV1
- type: object
- properties:
- value:
- type: number
- format: double
- description: >-
- The numeric value of the score. Equals 1 for "True" and 0 for "False"
- stringValue:
- type: string
- description: >-
- The string representation of the score value. Is inferred from the
- numeric value and equals "True" or "False"
- required:
- - value
- - stringValue
allOf:
- - $ref: '#/components/schemas/BaseScoreV1'
- CategoricalScoreV1:
- title: CategoricalScoreV1
+ - $ref: '#/components/schemas/EvaluatorOutputDefinitionBase'
+ PublicEvaluatorBooleanScore:
+ title: PublicEvaluatorBooleanScore
type: object
properties:
- value:
- type: number
- format: double
- description: >-
- Represents the numeric category mapping of the stringValue. If no
- config is linked, defaults to 0.
- stringValue:
+ dataType:
type: string
- description: >-
- The string representation of the score value. If no config is
- linked, can be any string. Otherwise, must map to a config category
+ const: BOOLEAN
+ description: Boolean score output.
required:
- - value
- - stringValue
+ - dataType
allOf:
- - $ref: '#/components/schemas/BaseScoreV1'
- TextScoreV1:
- title: TextScoreV1
+ - $ref: '#/components/schemas/EvaluatorOutputDefinitionBase'
+ PublicEvaluatorCategoricalScore:
+ title: PublicEvaluatorCategoricalScore
type: object
properties:
- stringValue:
+ dataType:
type: string
- description: The text content of the score (1-500 characters)
+ const: CATEGORICAL
+ description: Categorical score output.
+ categories:
+ type: array
+ items:
+ type: string
+ description: Allowed category values. At least two unique values are required.
+ shouldAllowMultipleMatches:
+ type: boolean
+ description: Whether the evaluator may return more than one category.
required:
- - stringValue
+ - dataType
+ - categories
+ - shouldAllowMultipleMatches
allOf:
- - $ref: '#/components/schemas/BaseScoreV1'
- ScoreV1:
- title: ScoreV1
+ - $ref: '#/components/schemas/EvaluatorOutputDefinitionBase'
+ PublicEvaluatorOutputDefinition:
+ title: PublicEvaluatorOutputDefinition
oneOf:
- type: object
allOf:
@@ -10084,18 +12448,7 @@ components:
type: string
enum:
- NUMERIC
- - $ref: '#/components/schemas/NumericScoreV1'
- required:
- - dataType
- - type: object
- allOf:
- - type: object
- properties:
- dataType:
- type: string
- enum:
- - CATEGORICAL
- - $ref: '#/components/schemas/CategoricalScoreV1'
+ - $ref: '#/components/schemas/PublicEvaluatorNumericScore'
required:
- dataType
- type: object
@@ -10106,7 +12459,7 @@ components:
type: string
enum:
- BOOLEAN
- - $ref: '#/components/schemas/BooleanScoreV1'
+ - $ref: '#/components/schemas/PublicEvaluatorBooleanScore'
required:
- dataType
- type: object
@@ -10116,1236 +12469,1588 @@ components:
dataType:
type: string
enum:
- - TEXT
- - $ref: '#/components/schemas/TextScoreV1'
+ - CATEGORICAL
+ - $ref: '#/components/schemas/PublicEvaluatorCategoricalScore'
required:
- dataType
- BaseScore:
- title: BaseScore
+ description: >-
+ Flat evaluator output definition returned by the public API.
+
+
+ This response always includes `dataType` and never includes an internal
+ output-definition `version`.
+
+ Optional empty descriptions from legacy definitions are omitted.
+ EvaluationRuleStringFilterOperator:
+ title: EvaluationRuleStringFilterOperator
+ type: string
+ enum:
+ - '='
+ - contains
+ - does not contain
+ - starts with
+ - ends with
+ EvaluationRuleNumberFilterOperator:
+ title: EvaluationRuleNumberFilterOperator
+ type: string
+ enum:
+ - '='
+ - '>'
+ - <
+ - '>='
+ - <=
+ EvaluationRuleOptionsFilterOperator:
+ title: EvaluationRuleOptionsFilterOperator
+ type: string
+ enum:
+ - any of
+ - none of
+ EvaluationRuleArrayOptionsFilterOperator:
+ title: EvaluationRuleArrayOptionsFilterOperator
+ type: string
+ enum:
+ - any of
+ - none of
+ - all of
+ EvaluationRuleBooleanFilterOperator:
+ title: EvaluationRuleBooleanFilterOperator
+ type: string
+ enum:
+ - '='
+ - <>
+ EvaluationRuleNullFilterOperator:
+ title: EvaluationRuleNullFilterOperator
+ type: string
+ enum:
+ - is null
+ - is not null
+ DateTimeEvaluationRuleFilter:
+ title: DateTimeEvaluationRuleFilter
type: object
properties:
- id:
+ column:
type: string
- traceId:
+ description: Column to filter on.
+ operator:
+ $ref: '#/components/schemas/EvaluationRuleNumberFilterOperator'
+ description: Comparison operator for datetime values.
+ value:
type: string
- nullable: true
- description: The trace ID associated with the score
- sessionId:
+ format: date-time
+ description: Datetime value to compare against.
+ required:
+ - column
+ - operator
+ - value
+ StringEvaluationRuleFilter:
+ title: StringEvaluationRuleFilter
+ type: object
+ properties:
+ column:
type: string
- nullable: true
- description: The session ID associated with the score
- observationId:
+ description: Column to filter on.
+ operator:
+ $ref: '#/components/schemas/EvaluationRuleStringFilterOperator'
+ value:
type: string
- nullable: true
- description: The observation ID associated with the score
- datasetRunId:
+ required:
+ - column
+ - operator
+ - value
+ NumberEvaluationRuleFilter:
+ title: NumberEvaluationRuleFilter
+ type: object
+ properties:
+ column:
type: string
- nullable: true
- description: The dataset run ID associated with the score
- name:
+ description: Column to filter on.
+ operator:
+ $ref: '#/components/schemas/EvaluationRuleNumberFilterOperator'
+ value:
+ type: number
+ format: double
+ required:
+ - column
+ - operator
+ - value
+ StringOptionsEvaluationRuleFilter:
+ title: StringOptionsEvaluationRuleFilter
+ type: object
+ properties:
+ column:
type: string
- source:
- $ref: '#/components/schemas/ScoreSource'
- timestamp:
+ description: Column to filter on.
+ operator:
+ $ref: '#/components/schemas/EvaluationRuleOptionsFilterOperator'
+ value:
+ type: array
+ items:
+ type: string
+ description: One or more allowed string values.
+ required:
+ - column
+ - operator
+ - value
+ ArrayOptionsEvaluationRuleFilter:
+ title: ArrayOptionsEvaluationRuleFilter
+ type: object
+ properties:
+ column:
type: string
- format: date-time
- createdAt:
+ description: Column to filter on.
+ operator:
+ $ref: '#/components/schemas/EvaluationRuleArrayOptionsFilterOperator'
+ value:
+ type: array
+ items:
+ type: string
+ description: One or more array elements to match.
+ required:
+ - column
+ - operator
+ - value
+ StringObjectEvaluationRuleFilter:
+ title: StringObjectEvaluationRuleFilter
+ type: object
+ properties:
+ column:
type: string
- format: date-time
- updatedAt:
+ description: >-
+ Object-valued column to filter on. Currently only `metadata` is
+ supported.
+ key:
type: string
- format: date-time
- authorUserId:
+ description: Top-level key inside the object-valued column to filter on.
+ operator:
+ $ref: '#/components/schemas/EvaluationRuleStringFilterOperator'
+ value:
type: string
- nullable: true
- description: The user ID of the author
- comment:
+ required:
+ - column
+ - key
+ - operator
+ - value
+ NumberObjectEvaluationRuleFilter:
+ title: NumberObjectEvaluationRuleFilter
+ type: object
+ properties:
+ column:
type: string
- nullable: true
- description: Comment on the score
- metadata:
- description: Metadata associated with the score
- configId:
+ description: Object-valued column to filter on.
+ key:
type: string
- nullable: true
- description: >-
- Reference a score config on a score. When set, config and score name
- must be equal and value must comply to optionally defined numerical
- range
- queueId:
+ description: Key inside the object-valued column to filter on.
+ operator:
+ $ref: '#/components/schemas/EvaluationRuleNumberFilterOperator'
+ value:
+ type: number
+ format: double
+ required:
+ - column
+ - key
+ - operator
+ - value
+ CategoryOptionsEvaluationRuleFilter:
+ title: CategoryOptionsEvaluationRuleFilter
+ type: object
+ properties:
+ column:
type: string
- nullable: true
- description: >-
- The annotation queue referenced by the score. Indicates if score was
- initially created while processing annotation queue.
- environment:
+ description: Object-valued column to filter on.
+ key:
type: string
- description: >-
- The environment from which this score originated. Can be any
- lowercase alphanumeric string with hyphens and underscores that does
- not start with 'langfuse'.
+ description: Key inside the object-valued column to filter on.
+ operator:
+ $ref: '#/components/schemas/EvaluationRuleOptionsFilterOperator'
+ value:
+ type: array
+ items:
+ type: string
required:
- - id
- - name
- - source
- - timestamp
- - createdAt
- - updatedAt
- - authorUserId
- - comment
- - metadata
- - configId
- - queueId
- - environment
- NumericScore:
- title: NumericScore
+ - column
+ - key
+ - operator
+ - value
+ BooleanEvaluationRuleFilter:
+ title: BooleanEvaluationRuleFilter
type: object
properties:
+ column:
+ type: string
+ description: Column to filter on.
+ operator:
+ $ref: '#/components/schemas/EvaluationRuleBooleanFilterOperator'
value:
- type: number
- format: double
- description: The numeric value of the score
+ type: boolean
required:
+ - column
+ - operator
- value
- allOf:
- - $ref: '#/components/schemas/BaseScore'
- BooleanScore:
- title: BooleanScore
+ NullEvaluationRuleFilter:
+ title: NullEvaluationRuleFilter
type: object
properties:
+ column:
+ type: string
+ description: Column to filter on, for example `parentObservationId`.
+ operator:
+ $ref: '#/components/schemas/EvaluationRuleNullFilterOperator'
value:
- type: number
- format: double
- description: >-
- The numeric value of the score. Equals 1 for "True" and 0 for "False"
- stringValue:
type: string
+ const: ''
description: >-
- The string representation of the score value. Is inferred from the
- numeric value and equals "True" or "False"
+ Required empty-string placeholder used by the runtime filter
+ contract.
required:
+ - column
+ - operator
- value
- - stringValue
- allOf:
- - $ref: '#/components/schemas/BaseScore'
- CategoricalScore:
- title: CategoricalScore
+ PromptVariableMappingInput:
+ title: PromptVariableMappingInput
type: object
+ description: >-
+ Connects one prompt variable to data from an observation or experiment.
+
+
+ Manual mappings are used for `llm_as_judge` evaluators. `code`
+ evaluators use a fixed runtime mapping managed by Langfuse.
+
+
+ How to build a valid mapping list:
+
+ 1. Create the evaluator or fetch it with `GET /evaluators/{id}`.
+
+ 2. Read the evaluator `variables` array.
+
+ 3. Add exactly one mapping object for each variable in that array.
+
+ 4. Use the variable name exactly as returned, without braces such as
+ `{{` or `}}`.
+
+ 5. Choose the source field that should populate the variable.
+
+
+ `jsonPath` is optional. Use it only when the selected source is a JSON
+ object and you want to extract one nested field before inserting it into
+ the evaluator prompt.
+
+
+ Invalid, missing, or duplicate mappings return a validation error.
+ Malformed JSONPath expressions are also rejected.
properties:
- value:
- type: number
- format: double
+ variable:
+ type: string
description: >-
- Represents the numeric category mapping of the stringValue. If no
- config is linked, defaults to 0.
- stringValue:
+ Prompt variable name without braces.
+
+
+ Example: for the prompt `Judge {{input}} against {{output}}`, use
+ `input` and `output`.
+ source:
+ $ref: '#/components/schemas/PromptVariableMappingSource'
+ description: >-
+ Source field that should populate the prompt variable.
+
+
+ Available sources are `input`, `output`, `metadata`, `tool_calls`,
+ `expected_output`, and `experiment_item_metadata`.
+ jsonPath:
type: string
+ nullable: true
description: >-
- The string representation of the score value. If no config is
- linked, can be any string. Otherwise, must map to a config category
+ Optional JSONPath selector applied to the selected source before it
+ is passed to the evaluator prompt.
+
+
+ Requirements:
+
+ - Must start with `$`
+
+ - Must be a syntactically valid JSONPath expression
+
+ - Most useful with `source=metadata`
required:
- - value
- - stringValue
- allOf:
- - $ref: '#/components/schemas/BaseScore'
- CorrectionScore:
- title: CorrectionScore
+ - variable
+ - source
+ PromptVariableMappingRead:
+ title: PromptVariableMappingRead
type: object
+ description: |-
+ Connects one prompt variable to source data.
+
+ `source` is `null` when the mapping is incomplete.
properties:
- value:
- type: number
- format: double
- description: The numeric value of the score. Always 0 for correction scores.
- stringValue:
+ variable:
type: string
- description: The string representation of the correction content
+ description: Prompt variable name without braces.
+ source:
+ type: string
+ nullable: true
+ description: >-
+ Stored source field populating the variable, or `null` when the
+ mapping is incomplete.
+ jsonPath:
+ type: string
+ nullable: true
+ description: Optional JSONPath selector applied to the selected source.
required:
- - value
- - stringValue
- allOf:
- - $ref: '#/components/schemas/BaseScore'
- TextScore:
- title: TextScore
+ - variable
+ - source
+ PromptVariableMapping:
+ title: PromptVariableMapping
+ oneOf:
+ - $ref: '#/components/schemas/PromptVariableMappingRead'
+ - $ref: '#/components/schemas/LegacyPromptVariableMapping'
+ description: Connects one prompt variable to source data.
+ LegacyPromptVariableMapping:
+ title: LegacyPromptVariableMapping
type: object
+ description: >-
+ **Deprecated:** Connects one prompt variable to data from a legacy trace
+ or dataset evaluation rule.
+
+
+ `langfuseObject` selects the object kind. `objectName` separately
+ selects a named observation and is `null` for trace and dataset-item
+ mappings.
properties:
- stringValue:
+ mappingType:
type: string
- description: The text content of the score (1-500 characters)
+ const: legacy
+ description: Explicitly marks this as a legacy mapping.
+ variable:
+ type: string
+ description: Prompt variable name without braces.
+ langfuseObject:
+ $ref: '#/components/schemas/LegacyEvaluationObject'
+ description: Legacy object kind selected as the mapping source.
+ objectName:
+ type: string
+ nullable: true
+ description: >-
+ Observation name to match, or `null` when `langfuseObject` is
+ `trace` or `dataset_item`.
+ source:
+ type: string
+ description: Field selected from the legacy object.
+ jsonPath:
+ type: string
+ nullable: true
+ description: Optional JSONPath selector applied to the selected field.
required:
- - stringValue
- allOf:
- - $ref: '#/components/schemas/BaseScore'
- Score:
- title: Score
+ - mappingType
+ - variable
+ - langfuseObject
+ - objectName
+ - source
+ LegacyEvaluationObject:
+ title: LegacyEvaluationObject
+ type: string
+ enum:
+ - trace
+ - span
+ - generation
+ - event
+ - agent
+ - tool
+ - chain
+ - retriever
+ - evaluator
+ - embedding
+ - guardrail
+ - dataset_item
+ description: >-
+ **Deprecated:** Legacy Langfuse object kind used by trace and dataset
+ evaluation rules.
+ EvaluationRuleFilter:
+ title: EvaluationRuleFilter
oneOf:
- type: object
allOf:
- type: object
properties:
- dataType:
+ type:
type: string
enum:
- - NUMERIC
- - $ref: '#/components/schemas/NumericScore'
+ - datetime
+ - $ref: '#/components/schemas/DateTimeEvaluationRuleFilter'
required:
- - dataType
+ - type
- type: object
allOf:
- type: object
properties:
- dataType:
+ type:
type: string
enum:
- - CATEGORICAL
- - $ref: '#/components/schemas/CategoricalScore'
+ - string
+ - $ref: '#/components/schemas/StringEvaluationRuleFilter'
required:
- - dataType
+ - type
- type: object
allOf:
- type: object
properties:
- dataType:
+ type:
type: string
enum:
- - BOOLEAN
- - $ref: '#/components/schemas/BooleanScore'
+ - number
+ - $ref: '#/components/schemas/NumberEvaluationRuleFilter'
required:
- - dataType
+ - type
- type: object
allOf:
- type: object
properties:
- dataType:
+ type:
type: string
enum:
- - CORRECTION
- - $ref: '#/components/schemas/CorrectionScore'
+ - stringOptions
+ - $ref: '#/components/schemas/StringOptionsEvaluationRuleFilter'
required:
- - dataType
+ - type
- type: object
allOf:
- type: object
properties:
- dataType:
+ type:
type: string
enum:
- - TEXT
- - $ref: '#/components/schemas/TextScore'
+ - categoryOptions
+ - $ref: '#/components/schemas/CategoryOptionsEvaluationRuleFilter'
required:
- - dataType
- properties:
- _deprecation:
- $ref: '#/components/schemas/Deprecation'
- nullable: true
- CreateScoreValue:
- title: CreateScoreValue
- oneOf:
- - type: number
- format: double
- - type: string
- description: >-
- The value of the score. Must be passed as string for categorical and
- text scores, and numeric for boolean and numeric scores
- Comment:
- title: Comment
- type: object
- properties:
- id:
- type: string
- projectId:
- type: string
- createdAt:
- type: string
- format: date-time
- updatedAt:
- type: string
- format: date-time
- objectType:
- $ref: '#/components/schemas/CommentObjectType'
- objectId:
- type: string
- content:
- type: string
- authorUserId:
- type: string
- nullable: true
- description: The user ID of the comment author
- required:
- - id
- - projectId
- - createdAt
- - updatedAt
- - objectType
- - objectId
- - content
- Dataset:
- title: Dataset
- type: object
- properties:
- id:
- type: string
- name:
- type: string
- description:
- type: string
- nullable: true
- description: Description of the dataset
- metadata:
- description: Metadata associated with the dataset
- inputSchema:
- nullable: true
- description: JSON Schema for validating dataset item inputs
- expectedOutputSchema:
- nullable: true
- description: JSON Schema for validating dataset item expected outputs
- projectId:
- type: string
- createdAt:
- type: string
- format: date-time
- updatedAt:
- type: string
- format: date-time
- required:
- - id
- - name
- - description
- - metadata
- - inputSchema
- - expectedOutputSchema
- - projectId
- - createdAt
- - updatedAt
- DatasetItem:
- title: DatasetItem
+ - type
+ - type: object
+ allOf:
+ - type: object
+ properties:
+ type:
+ type: string
+ enum:
+ - arrayOptions
+ - $ref: '#/components/schemas/ArrayOptionsEvaluationRuleFilter'
+ required:
+ - type
+ - type: object
+ allOf:
+ - type: object
+ properties:
+ type:
+ type: string
+ enum:
+ - stringObject
+ - $ref: '#/components/schemas/StringObjectEvaluationRuleFilter'
+ required:
+ - type
+ - type: object
+ allOf:
+ - type: object
+ properties:
+ type:
+ type: string
+ enum:
+ - numberObject
+ - $ref: '#/components/schemas/NumberObjectEvaluationRuleFilter'
+ required:
+ - type
+ - type: object
+ allOf:
+ - type: object
+ properties:
+ type:
+ type: string
+ enum:
+ - boolean
+ - $ref: '#/components/schemas/BooleanEvaluationRuleFilter'
+ required:
+ - type
+ - type: object
+ allOf:
+ - type: object
+ properties:
+ type:
+ type: string
+ enum:
+ - 'null'
+ - $ref: '#/components/schemas/NullEvaluationRuleFilter'
+ required:
+ - type
+ description: >-
+ One filter condition used to decide whether a live-ingested observation
+ should be evaluated.
+
+
+ All filters must be satisfied for the rule to run. Experiment scope is
+ expressed with filters rather than a separate target field:
+
+ - `isExperimentItemRootSpan = true` limits execution to experiment item
+ roots.
+
+ - `datasetId` limits execution to experiments for the selected datasets.
+ Use dataset IDs from `GET /api/public/v2/datasets`.
+
+
+ Pick the filter `type` first, as it determines the required fields and
+ value shape. Use `key` only for object filters such as `metadata`.
+
+
+ Operator quick reference by filter `type`:
+
+ - `string`: `"="`, `contains`, `does not contain`, `starts with`, `ends
+ with`
+
+ - `number`: `"="`, `">"`, `"<"`, `">="`, `"<="`
+
+ - `datetime`: `"="`, `">"`, `"<"`, `">="`, `"<="`
+
+ - `stringOptions`: `any of`, `none of`
+
+ - `arrayOptions`: `any of`, `none of`, `all of`
+
+ - `stringObject`: same operators as `string`
+
+ - `boolean`: `"="`, `"<>"`
+
+ - `null`: `is null`, `is not null`
+
+
+ Supported columns. Each column accepts exactly one filter `type`;
+ sending a column with a different `type` is rejected with `400`.
+
+ - `type`: `stringOptions`, values `SPAN`, `EVENT`, `GENERATION`,
+ `AGENT`, `TOOL`, `CHAIN`, `RETRIEVER`, `EVALUATOR`, `EMBEDDING`,
+ `GUARDRAIL`
+
+ - `name`: `stringOptions`
+
+ - `environment`: `stringOptions`
+
+ - `level`: `stringOptions`, values `DEBUG`, `DEFAULT`, `WARNING`,
+ `ERROR`
+
+ - `version`: `string`
+
+ - `traceName`: `stringOptions`
+
+ - `userId`: `string`
+
+ - `sessionId`: `string`
+
+ - `tags`: `arrayOptions`
+
+ - `metadata`: `stringObject`, requires `key`
+
+ - `isRootObservation`: `boolean`; true when the observation has no
+ parent or is explicitly marked as an application root
+
+ - `parentObservationId`: `null`
+
+ - `experimentId`: `stringOptions`
+
+ - `isExperimentItemRootSpan`: `boolean`
+
+ - `calledToolNames`: `arrayOptions`
+
+ - `toolCalls`: `number`, the number of tool calls on the observation
+
+ - `datasetId`: `stringOptions`. Use dataset `id` values from `GET
+ /api/public/v2/datasets`, not dataset names.
+ EvaluationRuleReadFilterBase:
+ title: EvaluationRuleReadFilterBase
type: object
+ description: Stored filter returned verbatim by evaluation-rule read endpoints.
properties:
- id:
+ type:
type: string
- status:
- $ref: '#/components/schemas/DatasetStatus'
- input:
- description: Input data for the dataset item
- expectedOutput:
- description: Expected output for the dataset item
- metadata:
- description: Metadata associated with the dataset item
- sourceTraceId:
+ description: >-
+ Stored filter type. This is not broken down into separate public
+ response types.
+ column:
type: string
- nullable: true
- description: The trace ID that sourced this dataset item
- sourceObservationId:
+ operator:
type: string
+ value:
nullable: true
- description: The observation ID that sourced this dataset item
- datasetId:
- type: string
- datasetName:
- type: string
- createdAt:
- type: string
- format: date-time
- updatedAt:
- type: string
- format: date-time
- mediaReferences:
- type: array
- items:
- $ref: '#/components/schemas/DatasetItemMediaReference'
- description: >-
- Resolved Langfuse media references found in input, expectedOutput,
- and metadata.
+ description: Stored filter value. Its shape depends on the filter type.
required:
- - id
- - status
- - input
- - expectedOutput
- - metadata
- - sourceTraceId
- - sourceObservationId
- - datasetId
- - datasetName
- - createdAt
- - updatedAt
- - mediaReferences
- DatasetItemMediaReference:
- title: DatasetItemMediaReference
+ - type
+ - column
+ - operator
+ EvaluationRuleReadFilterWithKey:
+ title: EvaluationRuleReadFilterWithKey
type: object
properties:
- field:
- $ref: '#/components/schemas/DatasetItemMediaReferenceField'
- description: The dataset item field containing the reference
- referenceString:
- type: string
- description: >-
- The Langfuse media reference string, e.g.
- `@@@langfuseMedia:type=image/png|id=...|source=bytes@@@`
- jsonPath:
+ key:
type: string
- description: >-
- JSONPath of the string holding the reference within the field, e.g.
- `$['image']`
- media:
- $ref: '#/components/schemas/DatasetItemMediaReferenceMedia'
- description: The resolved media record.
+ description: Stored object or legacy filter key.
required:
- - field
- - referenceString
- - jsonPath
- - media
- DatasetItemMediaReferenceField:
- title: DatasetItemMediaReferenceField
+ - key
+ allOf:
+ - $ref: '#/components/schemas/EvaluationRuleReadFilterBase'
+ EvaluationRuleReadFilter:
+ title: EvaluationRuleReadFilter
+ oneOf:
+ - $ref: '#/components/schemas/EvaluationRuleReadFilterWithKey'
+ - $ref: '#/components/schemas/EvaluationRuleReadFilterBase'
+ description: >-
+ Filter returned verbatim by evaluation-rule read endpoints.
+
+
+ Filters with a stored `key` use the keyed shape. All other filters use
+ the base shape. Response filters are intentionally not broken down by
+ their internal `type` value.
+ PublicApiErrorCode:
+ title: PublicApiErrorCode
type: string
enum:
- - input
- - expectedOutput
- - metadata
- DatasetItemMediaReferenceMedia:
- title: DatasetItemMediaReferenceMedia
+ - authentication_failed
+ - access_denied
+ - invalid_request
+ - invalid_query
+ - invalid_body
+ - resource_not_found
+ - conflict
+ - rate_limited
+ - method_not_allowed
+ - internal_error
+ description: >-
+ Stable machine-readable error code returned by the evaluators and
+ evaluation-rules API.
+ PublicApiValidationIssue:
+ title: PublicApiValidationIssue
type: object
+ description: One request validation issue.
properties:
- mediaId:
- type: string
- description: The unique langfuse identifier of the media record
- contentType:
- type: string
- description: The MIME type of the media record
- contentLength:
- type: integer
- description: The size of the media record in bytes
- url:
+ code:
type: string
- description: The signed download URL of the media record
- urlExpiry:
+ description: Validator issue code.
+ message:
type: string
- description: The expiry date and time of the download URL
+ description: Human-readable explanation.
+ path:
+ type: array
+ items: {}
+ description: Path to the invalid request field.
required:
- - mediaId
- - contentType
- - contentLength
- - url
- - urlExpiry
- DatasetRunItem:
- title: DatasetRunItem
+ - code
+ - message
+ - path
+ PublicApiErrorDetails:
+ title: PublicApiErrorDetails
type: object
+ description: Optional structured context for validation and rate-limit errors.
properties:
- id:
- type: string
- datasetRunId:
- type: string
- datasetRunName:
- type: string
- datasetItemId:
- type: string
- traceId:
- type: string
- observationId:
- type: string
+ issues:
+ type: array
+ items:
+ $ref: '#/components/schemas/PublicApiValidationIssue'
nullable: true
- description: The observation ID associated with this run item
- createdAt:
- type: string
- format: date-time
- updatedAt:
+ description: Validation issues for an invalid request body or query.
+ retryAfterSeconds:
+ type: integer
+ nullable: true
+ description: Number of seconds to wait before retrying a rate-limited request.
+ limit:
+ type: integer
+ nullable: true
+ description: Rate-limit request allowance.
+ remaining:
+ type: integer
+ nullable: true
+ description: Remaining requests in the current rate-limit window.
+ resetAt:
type: string
format: date-time
- required:
- - id
- - datasetRunId
- - datasetRunName
- - datasetItemId
- - traceId
- - observationId
- - createdAt
- - updatedAt
- DatasetRun:
- title: DatasetRun
+ nullable: true
+ description: Time when the current rate-limit window resets.
+ PublicApiError:
+ title: PublicApiError
type: object
+ description: >-
+ Standard error response for the stable evaluators and evaluation-rules
+ API.
+
+
+ Use the HTTP status for the broad failure class and `code` for
+ programmatic handling. `details` is included when field-level validation
+ or retry information is available.
properties:
- id:
- type: string
- description: Unique identifier of the dataset run
- name:
- type: string
- description: Name of the dataset run
- description:
+ message:
type: string
+ description: Human-readable description of the failure.
+ code:
+ $ref: '#/components/schemas/PublicApiErrorCode'
+ description: Stable machine-readable error code.
+ details:
+ $ref: '#/components/schemas/PublicApiErrorDetails'
nullable: true
- description: Description of the run
- metadata:
- description: Metadata of the dataset run
- datasetId:
- type: string
- description: Id of the associated dataset
- datasetName:
- type: string
- description: Name of the associated dataset
- createdAt:
- type: string
- format: date-time
- description: The date and time when the dataset run was created
- updatedAt:
+ description: Optional structured validation or rate-limit context.
+ required:
+ - message
+ - code
+ EvaluationRuleEvaluatorAssignmentInput:
+ title: EvaluationRuleEvaluatorAssignmentInput
+ type: object
+ properties:
+ evaluatorId:
type: string
- format: date-time
- description: The date and time when the dataset run was last updated
+ description: >-
+ Stable evaluator identifier. The rule automatically uses that
+ evaluator's latest version.
+ variableMapping:
+ type: array
+ items:
+ $ref: '#/components/schemas/PromptVariableMappingInput'
+ nullable: true
+ description: >-
+ Rule-specific prompt-variable mapping.
+
+
+ Set to `null` or omit to inherit the evaluator's latest default
+ mapping. Code evaluators use the fixed runtime mapping and should
+ use `null`.
required:
- - id
- - name
- - description
- - metadata
- - datasetId
- - datasetName
- - createdAt
- - updatedAt
- DatasetRunWithItems:
- title: DatasetRunWithItems
+ - evaluatorId
+ EvaluatorAssignment:
+ title: EvaluatorAssignment
type: object
properties:
- datasetRunItems:
+ evaluatorId:
+ type: string
+ description: Stable identifier of the evaluator associated with this rule.
+ variableMapping:
type: array
items:
- $ref: '#/components/schemas/DatasetRunItem'
- _deprecation:
- $ref: '#/components/schemas/Deprecation'
+ $ref: '#/components/schemas/PromptVariableMapping'
nullable: true
+ description: >-
+ Stored rule-specific override, or `null` when the evaluator's latest
+ default mapping is inherited. Legacy mappings are returned as an
+ explicit `legacy` variant.
required:
- - datasetRunItems
- allOf:
- - $ref: '#/components/schemas/DatasetRun'
- Model:
- title: Model
+ - evaluatorId
+ - variableMapping
+ EvaluationRule:
+ title: EvaluationRule
type: object
description: >-
- Model definition used for transforming usage into USD cost and/or
- tokenization.
-
-
- Models can have either simple flat pricing or tiered pricing:
-
- - Flat pricing: Single price per usage type (legacy, but still
- supported)
-
- - Tiered pricing: Multiple pricing tiers with conditional matching based
- on usage patterns or observation attributes
-
-
- The pricing tiers approach is recommended for models with usage-based
- pricing variations.
+ Live evaluation rule for incoming observations.
- When using tiered pricing, the flat price fields (inputPrice,
- outputPrice, prices) are populated
- from the default tier for backward compatibility.
+ A rule determines which evaluators should be used, which observations
+ should trigger scoring, how often scoring should run, and which
+ observation fields should populate prompt variables.
properties:
id:
type: string
- modelName:
- type: string
- description: >-
- Name of the model definition. If multiple with the same name exist,
- they are applied in the following order: (1) custom over built-in,
- (2) newest according to startTime where
- model.startTime<observation.startTime
- matchPattern:
+ description: Stable evaluation-rule identifier.
+ name:
type: string
description: >-
- Regex pattern which matches this model definition to
- generation.model. Useful in case of fine-tuned models. If you want
- to exact match, use `(?i)^modelname$`
- startDate:
- type: string
- format: date-time
- nullable: true
- description: Apply only to generations which are newer than this ISO date.
- unit:
- $ref: '#/components/schemas/ModelUsageUnit'
- nullable: true
- description: Unit used by this model.
- inputPrice:
- type: number
- format: double
- nullable: true
- description: Deprecated. See 'prices' instead. Price (USD) per input unit
- outputPrice:
- type: number
- format: double
+ Human-readable rule name. This is independent from evaluator names
+ and does not need to be unique.
+ createdBy:
+ $ref: '#/components/schemas/Creator'
nullable: true
- description: Deprecated. See 'prices' instead. Price (USD) per output unit
- totalPrice:
+ description: User who created this rule, or `null` when no user can be resolved.
+ enabled:
+ type: boolean
+ description: Whether live execution is enabled for this rule.
+ sampling:
type: number
format: double
- nullable: true
- description: >-
- Deprecated. See 'prices' instead. Price (USD) per total unit. Cannot
- be set if input or output price is set.
- tokenizerId:
- type: string
- nullable: true
- description: >-
- Optional. Tokenizer to be applied to observations which match to
- this model. See docs for more details.
- tokenizerConfig:
- description: >-
- Optional. Configuration for the selected tokenizer. Needs to be
- JSON. See docs for more details.
- isLangfuseManaged:
- type: boolean
- createdAt:
- type: string
- format: date-time
- description: Timestamp when the model was created
- prices:
- type: object
- additionalProperties:
- $ref: '#/components/schemas/ModelPrice'
- description: >-
- Deprecated. Use 'pricingTiers' instead for models with usage-based
- pricing variations.
-
-
- This field shows prices by usage type from the default pricing tier.
- Maintained for backward compatibility.
+ description: |-
+ Fraction of matching observations that should be evaluated.
- If the model uses tiered pricing, this field will be populated from
- the default tier's prices.
- pricingTiers:
+ Must be between `0` and `1`.
+ - `1` evaluates every matching observation.
+ - `0.25` evaluates approximately 25% of matching observations.
+ filter:
type: array
items:
- $ref: '#/components/schemas/PricingTier'
+ $ref: '#/components/schemas/EvaluationRuleReadFilter'
description: >-
- Array of pricing tiers with conditional pricing based on usage
- thresholds or observation attributes.
-
-
- Pricing tiers enable accurate cost tracking for models that charge
- different rates based on usage patterns
-
- or request attributes (e.g., high-volume usage, large context
- windows, cached tokens, or service tiers).
-
-
- Each model must have exactly one default tier (isDefault=true,
- priority=0) that serves as a fallback.
-
- Additional conditional tiers can be defined with specific matching
- criteria.
-
-
- If this array is empty, the model uses legacy flat pricing from the
- inputPrice/outputPrice/totalPrice fields.
+ List of stored filter conditions returned verbatim. Filters with a
+ `key` use the keyed response shape; all others use the base shape.
+ These response shapes are not broken down by internal filter type.
+ An empty list matches every incoming object.
+ evaluatorAssignments:
+ type: array
+ items:
+ $ref: '#/components/schemas/EvaluatorAssignment'
+ description: Evaluators attached to this rule in deterministic assignment order.
+ createdAt:
+ type: string
+ format: date-time
+ description: Timestamp when the evaluation rule was created.
+ updatedAt:
+ type: string
+ format: date-time
+ description: Timestamp when the evaluation rule was last updated.
required:
- id
- - modelName
- - matchPattern
- - startDate
- - inputPrice
- - outputPrice
- - totalPrice
- - tokenizerId
- - tokenizerConfig
- - isLangfuseManaged
+ - name
+ - createdBy
+ - enabled
+ - sampling
+ - filter
+ - evaluatorAssignments
- createdAt
- - prices
- - pricingTiers
- ModelPrice:
- title: ModelPrice
+ - updatedAt
+ CreateEvaluationRuleRequest:
+ title: CreateEvaluationRuleRequest
type: object
properties:
- price:
+ name:
+ type: string
+ description: >-
+ Human-readable rule name. Names are not identifiers and do not need
+ to be unique.
+ enabled:
+ type: boolean
+ description: >-
+ Whether live execution should start immediately. Enabled rules
+ require at least one evaluator assignment.
+ sampling:
type: number
format: double
+ nullable: true
+ description: >-
+ Fraction of matching observations to evaluate. Omit this field to
+ use the default of `1`, which evaluates every match. `null` is not
+ accepted.
+ filter:
+ type: array
+ items:
+ $ref: '#/components/schemas/EvaluationRuleFilter'
+ nullable: true
+ description: >-
+ Conditions used to select observations. Defaults to an empty list,
+ which matches every incoming observation.
+ evaluatorAssignments:
+ type: array
+ items:
+ $ref: '#/components/schemas/EvaluationRuleEvaluatorAssignmentInput'
+ description: >-
+ Evaluators to attach to this rule. Disabled rules may use an empty
+ list as a draft.
required:
- - price
- PricingTierCondition:
- title: PricingTierCondition
- oneOf:
- - $ref: '#/components/schemas/PricingTierUsageCondition'
- - $ref: '#/components/schemas/PricingTierAttributeCondition'
- description: >-
- Condition for matching a pricing tier against usage details or
- observation attributes.
-
-
- Usage-detail conditions treat usageDetailPattern as a regex, sum all
- matching usage values, and compare the sum to the numeric value.
- Model-parameter and metadata conditions match an exact top-level key
- against one or more string values.
- PricingTierUsageCondition:
- title: PricingTierUsageCondition
+ - name
+ - enabled
+ - evaluatorAssignments
+ UpdateEvaluationRuleRequest:
+ title: UpdateEvaluationRuleRequest
type: object
- description: Condition that sums usage details whose keys match a regex.
+ description: At least one field must be provided.
properties:
- usageDetailPattern:
+ name:
type: string
- description: Regex pattern matched against usage detail keys.
- operator:
- $ref: '#/components/schemas/PricingTierOperator'
- value:
+ nullable: true
+ description: New human-readable rule name.
+ enabled:
+ type: boolean
+ nullable: true
+ description: New desired live-execution state.
+ sampling:
type: number
format: double
- description: Numeric threshold for the summed matching usage values.
- caseSensitive:
- type: boolean
- description: Whether the usage-detail regex is case-sensitive.
- required:
- - usageDetailPattern
- - operator
- - value
- - caseSensitive
- PricingTierAttributeCondition:
- title: PricingTierAttributeCondition
- type: object
- description: >-
- Condition that matches any configured value for a top-level observation
- attribute.
- properties:
- source:
- $ref: '#/components/schemas/PricingTierAttributeSource'
- description: Observation attribute object evaluated by this condition.
- key:
- type: string
- description: Exact top-level attribute key.
- operator:
- type: string
- const: in
- description: Membership operator.
- values:
+ nullable: true
+ description: >-
+ New fraction of matching observations to evaluate. Omit to keep the
+ current value.
+ filter:
+ type: array
+ items:
+ $ref: '#/components/schemas/EvaluationRuleFilter'
+ nullable: true
+ description: >-
+ Complete replacement filter list. An empty list matches every
+ incoming observation.
+ evaluatorAssignments:
+ type: array
+ items:
+ $ref: '#/components/schemas/EvaluationRuleEvaluatorAssignmentInput'
+ nullable: true
+ description: >-
+ Complete replacement assignment list. An empty list disables the
+ rule.
+ EvaluationRulesPage:
+ title: EvaluationRulesPage
+ type: object
+ properties:
+ data:
type: array
items:
- type: string
- description: Accepted string attribute values. At least one value is required.
+ $ref: '#/components/schemas/EvaluationRule'
+ description: Evaluation rules for this page.
+ meta:
+ $ref: '#/components/schemas/CursorMeta'
+ description: Cursor pagination metadata.
required:
- - source
- - key
- - operator
- - values
- PricingTierAttributeSource:
- title: PricingTierAttributeSource
- type: string
- enum:
- - model_parameters
- - metadata
- PricingTierConditionInput:
- title: PricingTierConditionInput
- oneOf:
- - $ref: '#/components/schemas/PricingTierUsageConditionInput'
- - $ref: '#/components/schemas/PricingTierAttributeCondition'
- description: Condition input for matching a pricing tier.
- PricingTierUsageConditionInput:
- title: PricingTierUsageConditionInput
+ - data
+ - meta
+ DeletedEvaluationRule:
+ title: DeletedEvaluationRule
type: object
- description: Input condition that sums usage details whose keys match a regex.
+ description: Confirmation returned after successful evaluation-rule deletion.
properties:
- usageDetailPattern:
+ id:
+ type: string
+ description: Identifier of the deleted evaluation rule.
+ required:
+ - id
+ Creator:
+ title: Creator
+ type: object
+ description: User who created the resource, or `null` when no user can be resolved.
+ properties:
+ id:
+ type: string
+ description: User identifier.
+ name:
type: string
- description: Regex pattern matched against usage detail keys.
- operator:
- $ref: '#/components/schemas/PricingTierOperator'
- value:
- type: number
- format: double
- description: Numeric threshold for the summed matching usage values.
- caseSensitive:
- type: boolean
nullable: true
- description: Whether the usage-detail regex is case-sensitive. Defaults to false.
+ description: User name, or `null` when unavailable.
required:
- - usageDetailPattern
- - operator
- - value
- PricingTier:
- title: PricingTier
+ - id
+ - name
+ EvaluatorModelConfig:
+ title: EvaluatorModelConfig
type: object
description: >-
- Pricing tier definition with conditional pricing based on usage
- thresholds or observation attributes.
-
-
- Pricing tiers enable accurate cost tracking for LLM providers that
- charge different rates based on usage patterns or request attributes.
-
- For example, some providers charge higher rates when context size
- exceeds certain thresholds or when an accelerated service tier is
- selected.
-
-
- How tier matching works:
-
- 1. Tiers are evaluated in ascending priority order (priority 1 before
- priority 2, etc.)
-
- 2. The first tier where ALL conditions match is selected
-
- 3. If no conditional tiers match, the default tier is used as a fallback
-
- 4. The default tier has priority 0 and no conditions
-
-
- Why priorities matter:
-
- - Lower priority numbers are evaluated first, allowing you to define
- specific cases before general ones
+ Explicit model configuration for an evaluator.
- - Example: Priority 1 for "high usage" (>200K tokens), Priority 2 for
- "medium usage" (>100K tokens), Priority 0 for default
-
- - Without proper ordering, a less specific condition might match before
- a more specific one
+ If the complete `modelConfig` is `null`, Langfuse uses the project's
+ default evaluation model.
- Every model must have exactly one default tier to ensure cost
- calculation always succeeds.
+ If provided, the model must be available to the project when the
+ evaluator or evaluation rule is enabled.
properties:
- id:
- type: string
- description: Unique identifier for the pricing tier
- name:
+ provider:
type: string
description: >-
- Name of the pricing tier for display and identification purposes.
+ Provider identifier, for example `openai` or `anthropic`.
- Examples: "Standard", "High Volume Tier", "Large Context", "Extended
- Context Tier"
- isDefault:
- type: boolean
+ To discover valid values for the current project, call `GET
+ /api/public/llm-connections` and use one of the returned `provider`
+ values.
+ model:
+ type: string
description: >-
- Whether this is the default tier. Every model must have exactly one
- default tier with priority 0 and no conditions.
-
-
- The default tier serves as a fallback when no conditional tiers
- match, ensuring cost calculation always succeeds.
-
- It typically represents the base pricing for standard usage
- patterns.
- priority:
+ Model identifier exposed by the provider, for example `gpt-4.1-mini`.
+ required:
+ - provider
+ - model
+ EvaluatorChatMessage:
+ title: EvaluatorChatMessage
+ type: object
+ description: One user chat message in an evaluator prompt.
+ properties:
+ role:
+ type: string
+ const: user
+ content:
+ type: string
+ description: Message content. Evaluator variables use `{{variable}}` syntax.
+ required:
+ - role
+ - content
+ EvaluatorChatPrompt:
+ title: EvaluatorChatPrompt
+ type: array
+ items:
+ $ref: '#/components/schemas/EvaluatorChatMessage'
+ description: A list containing exactly one user chat message.
+ EvaluatorChatPromptInput:
+ title: EvaluatorChatPromptInput
+ oneOf:
+ - type: string
+ - $ref: '#/components/schemas/EvaluatorChatPrompt'
+ description: >-
+ A user prompt string, or a list containing exactly one user chat message.
+ EvaluatorVersionBase:
+ title: EvaluatorVersionBase
+ type: object
+ properties:
+ id:
+ type: string
+ description: Stable identifier of this evaluator version.
+ version:
type: integer
+ description: Monotonically increasing evaluator version number.
+ createdAt:
+ type: string
+ format: date-time
+ description: Timestamp when this evaluator version was created.
+ createdBy:
+ $ref: '#/components/schemas/Creator'
+ nullable: true
description: >-
- Priority for tier matching evaluation. Lower numbers = higher
- priority (evaluated first).
-
-
- The default tier must always have priority 0. Conditional tiers
- should have priority 1, 2, 3, etc.
-
-
- Example ordering:
-
- - Priority 0: Default tier (no conditions, always matches as
- fallback)
-
- - Priority 1: High usage tier (e.g., >200K tokens)
-
- - Priority 2: Medium usage tier (e.g., >100K tokens)
-
-
- This ensures more specific conditions are checked before general
- ones.
- conditions:
+ User who created this version, or `null` when no user can be
+ resolved.
+ required:
+ - id
+ - version
+ - createdAt
+ - createdBy
+ LlmAsJudgeEvaluatorVersion:
+ title: LlmAsJudgeEvaluatorVersion
+ type: object
+ properties:
+ type:
+ type: string
+ const: llm_as_judge
+ description: Evaluator type.
+ prompt:
+ $ref: '#/components/schemas/EvaluatorChatPrompt'
+ description: The single user chat message used during evaluation.
+ variables:
type: array
items:
- $ref: '#/components/schemas/PricingTierCondition'
+ type: string
description: >-
- Array of conditions that must ALL be met for this tier to match (AND
- logic).
-
-
- The default tier must have an empty conditions array. Conditional
- tiers should have one or more conditions
-
- that define when this tier's pricing applies.
+ Variables extracted from the prompt and available for
+ evaluation-rule mappings.
- Multiple conditions enable complex matching scenarios (e.g., "high
- input tokens AND low output tokens").
- prices:
- type: object
- additionalProperties:
- type: number
- format: double
+ Every variable must be mapped exactly once when a rule provides an
+ explicit mapping.
+ variableMapping:
+ type: array
+ items:
+ $ref: '#/components/schemas/PromptVariableMappingRead'
+ nullable: true
description: >-
- Prices (USD) by usage type for this tier.
-
-
- Common usage types: "input", "output", "total", "request", "image"
-
- Prices are specified in USD per unit (e.g., per token, per request,
- per second).
+ Default variable mapping for this evaluator version, or `null` when
+ no default is configured.
+ modelConfig:
+ $ref: '#/components/schemas/EvaluatorModelConfig'
+ nullable: true
+ description: >-
+ Explicit model configuration, or `null` when the project's default
+ evaluation model is used.
+ outputDefinition:
+ $ref: '#/components/schemas/PublicEvaluatorOutputDefinition'
+ description: >-
+ Structured output schema returned by this evaluator.
- Example: {"input": 0.000003, "output": 0.000015} means $3 per
- million input tokens and $15 per million output tokens.
+ Responses include `dataType` and omit the internal output-definition
+ persistence version.
+ required:
+ - type
+ - prompt
+ - variables
+ - variableMapping
+ - modelConfig
+ - outputDefinition
+ allOf:
+ - $ref: '#/components/schemas/EvaluatorVersionBase'
+ CodeEvaluatorVersion:
+ title: CodeEvaluatorVersion
+ type: object
+ properties:
+ type:
+ type: string
+ const: code
+ description: Evaluator type.
+ sourceCode:
+ type: string
+ description: Source code executed for each matched observation.
+ sourceCodeLanguage:
+ $ref: '#/components/schemas/CodeEvaluatorSourceCodeLanguage'
+ description: Runtime language used to execute the source code.
+ required:
+ - type
+ - sourceCode
+ - sourceCodeLanguage
+ allOf:
+ - $ref: '#/components/schemas/EvaluatorVersionBase'
+ EvaluatorVersion:
+ title: EvaluatorVersion
+ oneOf:
+ - type: object
+ allOf:
+ - type: object
+ properties:
+ type:
+ type: string
+ enum:
+ - llm_as_judge
+ - $ref: '#/components/schemas/LlmAsJudgeEvaluatorVersion'
+ required:
+ - type
+ - type: object
+ allOf:
+ - type: object
+ properties:
+ type:
+ type: string
+ enum:
+ - code
+ - $ref: '#/components/schemas/CodeEvaluatorVersion'
+ required:
+ - type
+ EvaluationRuleAssignment:
+ title: EvaluationRuleAssignment
+ type: object
+ description: Evaluation-rule assignment associated with this evaluator.
+ properties:
+ evaluationRuleId:
+ type: string
+ description: Stable identifier of the assigned evaluation rule.
+ variableMappingOverride:
+ type: array
+ items:
+ $ref: '#/components/schemas/PromptVariableMapping'
+ nullable: true
+ description: >-
+ Rule-specific variable mapping override. Omitted when the
+ evaluator's latest default mapping is inherited. Legacy mappings use
+ the explicit `legacy` variant.
required:
- - id
- - name
- - isDefault
- - priority
- - conditions
- - prices
- PricingTierInput:
- title: PricingTierInput
+ - evaluationRuleId
+ EvaluatorBase:
+ title: EvaluatorBase
type: object
- description: >-
- Input schema for creating a pricing tier. The tier ID will be
- automatically generated server-side.
-
-
- When creating a model with pricing tiers:
-
- - Exactly one tier must have isDefault=true (the fallback tier)
-
- - The default tier must have priority=0 and conditions=[]
-
- - All tier names and priorities must be unique within the model
-
- - Each tier must define at least one price
-
-
- See PricingTier for detailed information about how tiers work and why
- they're useful.
properties:
+ id:
+ type: string
+ description: Stable identifier of this evaluator across all versions.
name:
type: string
description: >-
- Name of the pricing tier for display and identification purposes.
-
-
- Must be unique within the model. Common patterns: "Standard", "High
- Volume Tier", "Extended Context"
- isDefault:
- type: boolean
+ Human-readable evaluator name. Names are not identifiers and do not
+ need to be unique.
+ description:
+ type: string
+ nullable: true
+ description: Optional human-readable evaluator description.
+ createdBy:
+ $ref: '#/components/schemas/Creator'
nullable: true
description: >-
- Whether this is the default tier. Exactly one tier per model must be
- marked as default.
-
-
- Requirements for default tier:
-
- - Must have isDefault=true
-
- - Must have priority=0
-
- - Must have empty conditions array (conditions=[])
-
-
- The default tier acts as a fallback when no conditional tiers match.
- Defaults to false when omitted.
- priority:
- type: integer
+ User who created this evaluator, or `null` when no user can be
+ resolved.
+ status:
+ $ref: '#/components/schemas/EvaluatorStatus'
description: >-
- Priority for tier matching evaluation. Lower numbers = higher
- priority (evaluated first).
-
-
- Must be unique within the model. The default tier must have
- priority=0.
-
- Conditional tiers should use priority 1, 2, 3, etc. based on their
- specificity.
- conditions:
+ Effective evaluator status after Langfuse validates its runtime
+ configuration.
+ pausedAt:
+ type: string
+ format: date-time
+ nullable: true
+ description: Timestamp when the evaluator was paused, otherwise `null`.
+ pausedReason:
+ type: string
+ nullable: true
+ description: Machine-readable reason when `status=paused`, otherwise `null`.
+ pausedMessage:
+ type: string
+ nullable: true
+ description: Human-readable explanation when `status=paused`, otherwise `null`.
+ evaluationRuleAssignments:
type: array
items:
- $ref: '#/components/schemas/PricingTierConditionInput'
+ $ref: '#/components/schemas/EvaluationRuleAssignment'
description: >-
- Array of conditions that must ALL be met for this tier to match (AND
- logic).
-
-
- The default tier must have an empty array (conditions=[]).
-
- Conditional tiers should define one or more conditions that specify
- when this tier's pricing applies.
-
-
- Conditions can compare summed matching usage details to a numeric
- threshold, or exactly match a top-level model parameter or metadata
- value.
- prices:
- type: object
- additionalProperties:
- type: number
- format: double
+ All modern and legacy evaluation-rule assignments in
+ newest-assignment-first order, including each assigned rule ID and
+ its stored mapping.
+ createdAt:
+ type: string
+ format: date-time
+ description: Timestamp when the evaluator was created.
+ updatedAt:
+ type: string
+ format: date-time
+ description: Timestamp when the evaluator was last updated.
+ versionId:
+ type: string
+ description: Stable identifier of the latest evaluator version.
+ version:
+ type: integer
+ description: Monotonically increasing latest evaluator version number.
+ versionCreatedAt:
+ type: string
+ format: date-time
+ description: Timestamp when the latest evaluator version was created.
+ versionCreatedBy:
+ $ref: '#/components/schemas/Creator'
+ nullable: true
description: >-
- Prices (USD) by usage type for this tier. At least one price must be
- defined.
-
-
- Common usage types: "input", "output", "total", "request", "image"
-
- Prices are in USD per unit (e.g., per token).
-
-
- Example: {"input": 0.000003, "output": 0.000015} represents $3 per
- million input tokens and $15 per million output tokens.
+ User who created the latest evaluator version, or `null` when no
+ user can be resolved.
required:
+ - id
- name
- - priority
- - conditions
- - prices
- PricingTierOperator:
- title: PricingTierOperator
- type: string
- enum:
- - gt
- - gte
- - lt
- - lte
- - eq
- - neq
- description: Comparison operators for pricing tier conditions
- ModelUsageUnit:
- title: ModelUsageUnit
- type: string
- enum:
- - CHARACTERS
- - TOKENS
- - MILLISECONDS
- - SECONDS
- - IMAGES
- - REQUESTS
- description: Unit of usage in Langfuse
- ObservationLevel:
- title: ObservationLevel
- type: string
- enum:
- - DEBUG
- - DEFAULT
- - WARNING
- - ERROR
- MapValue:
- title: MapValue
- oneOf:
- - type: string
- nullable: true
- - type: integer
- nullable: true
- - type: number
- format: float
- nullable: true
- - type: boolean
- nullable: true
- - type: array
+ - description
+ - createdBy
+ - status
+ - pausedAt
+ - pausedReason
+ - pausedMessage
+ - evaluationRuleAssignments
+ - createdAt
+ - updatedAt
+ - versionId
+ - version
+ - versionCreatedAt
+ - versionCreatedBy
+ LlmAsJudgeEvaluator:
+ title: LlmAsJudgeEvaluator
+ type: object
+ properties:
+ type:
+ type: string
+ const: llm_as_judge
+ description: Evaluator type.
+ prompt:
+ $ref: '#/components/schemas/EvaluatorChatPrompt'
+ description: The single user chat message used by the latest evaluator version.
+ variables:
+ type: array
items:
type: string
+ description: >-
+ Variables extracted from the latest prompt and available for
+ evaluation-rule mappings.
+ variableMapping:
+ type: array
+ items:
+ $ref: '#/components/schemas/PromptVariableMappingRead'
nullable: true
- CommentObjectType:
- title: CommentObjectType
- type: string
- enum:
- - TRACE
- - OBSERVATION
- - SESSION
- - PROMPT
- DatasetStatus:
- title: DatasetStatus
- type: string
- enum:
- - ACTIVE
- - ARCHIVED
- ScoreSource:
- title: ScoreSource
- type: string
- enum:
- - ANNOTATION
- - API
- - EVAL
- ScoreConfigDataType:
- title: ScoreConfigDataType
- type: string
- enum:
- - NUMERIC
- - BOOLEAN
- - CATEGORICAL
- - TEXT
- ScoreDataType:
- title: ScoreDataType
+ description: >-
+ Default variable mapping for the latest version, or `null` when no
+ default is configured.
+ modelConfig:
+ $ref: '#/components/schemas/EvaluatorModelConfig'
+ nullable: true
+ description: >-
+ Explicit model configuration for the latest version, or `null` when
+ the project's default evaluation model is used.
+ outputDefinition:
+ $ref: '#/components/schemas/PublicEvaluatorOutputDefinition'
+ description: Structured output schema returned by the latest evaluator version.
+ required:
+ - type
+ - prompt
+ - variables
+ - variableMapping
+ - modelConfig
+ - outputDefinition
+ allOf:
+ - $ref: '#/components/schemas/EvaluatorBase'
+ CodeEvaluator:
+ title: CodeEvaluator
+ type: object
+ properties:
+ type:
+ type: string
+ const: code
+ description: Evaluator type.
+ sourceCode:
+ type: string
+ description: Source code executed by the latest evaluator version.
+ sourceCodeLanguage:
+ $ref: '#/components/schemas/CodeEvaluatorSourceCodeLanguage'
+ description: Runtime language used to execute the latest source code.
+ required:
+ - type
+ - sourceCode
+ - sourceCodeLanguage
+ allOf:
+ - $ref: '#/components/schemas/EvaluatorBase'
+ Evaluator:
+ title: Evaluator
+ oneOf:
+ - type: object
+ allOf:
+ - type: object
+ properties:
+ type:
+ type: string
+ enum:
+ - llm_as_judge
+ - $ref: '#/components/schemas/LlmAsJudgeEvaluator'
+ required:
+ - type
+ - type: object
+ allOf:
+ - type: object
+ properties:
+ type:
+ type: string
+ enum:
+ - code
+ - $ref: '#/components/schemas/CodeEvaluator'
+ required:
+ - type
+ description: >-
+ One evaluator that can be used for scoring.
+
+
+ An evaluator describes **how** to score data. Associated evaluation
+ rules describe **which** live objects should be evaluated.
+ EvaluatorStatus:
+ title: EvaluatorStatus
type: string
enum:
- - NUMERIC
- - BOOLEAN
- - CATEGORICAL
- - CORRECTION
- - TEXT
- DeleteDatasetItemResponse:
- title: DeleteDatasetItemResponse
- type: object
- properties:
- message:
- type: string
- description: Success message after deletion
- required:
- - message
- CreateDatasetItemRequest:
- title: CreateDatasetItemRequest
+ - active
+ - paused
+ description: >-
+ Effective evaluator runtime status.
+
+
+ - `active`: the evaluator can run.
+
+ - `paused`: Langfuse paused execution until the underlying issue is
+ resolved.
+ CreateLlmAsJudgeEvaluatorRequest:
+ title: CreateLlmAsJudgeEvaluatorRequest
type: object
properties:
- datasetName:
- type: string
- input:
- nullable: true
- expectedOutput:
- nullable: true
- metadata:
- nullable: true
- sourceTraceId:
+ name:
type: string
- nullable: true
- sourceObservationId:
+ description: >-
+ Human-readable evaluator name. Names are not identifiers and do not
+ need to be unique.
+ description:
type: string
nullable: true
- id:
+ description: Optional human-readable evaluator description.
+ type:
type: string
+ const: llm_as_judge
+ description: Evaluator type.
+ prompt:
+ $ref: '#/components/schemas/EvaluatorChatPromptInput'
+ description: >-
+ User prompt string shortcut or a list containing exactly one user
+ chat message.
+ modelConfig:
+ $ref: '#/components/schemas/EvaluatorModelConfig'
nullable: true
description: >-
- Dataset items are upserted on their id. Id needs to be unique
- (project-level), cannot be reused across datasets, and must be at
- most 255 characters.
- status:
- $ref: '#/components/schemas/DatasetStatus'
+ Explicit model configuration. Set to `null` or omit to use the
+ project's default evaluation model.
+ variableMapping:
+ type: array
+ items:
+ $ref: '#/components/schemas/PromptVariableMappingInput'
nullable: true
- description: Defaults to ACTIVE for newly created items
+ description: >-
+ Default prompt-variable mapping, or `null` when no default is
+ configured.
+ outputDefinition:
+ $ref: '#/components/schemas/EvaluatorOutputDefinition'
+ description: >-
+ Structured output schema returned by this evaluator. Do not include
+ an internal persistence version.
required:
- - datasetName
- PaginatedDatasetItems:
- title: PaginatedDatasetItems
+ - name
+ - type
+ - prompt
+ - outputDefinition
+ CreateCodeEvaluatorRequest:
+ title: CreateCodeEvaluatorRequest
type: object
properties:
- data:
- type: array
- items:
- $ref: '#/components/schemas/DatasetItem'
- meta:
- $ref: '#/components/schemas/utilsMetaResponse'
+ name:
+ type: string
+ description: >-
+ Human-readable evaluator name. Names are not identifiers and do not
+ need to be unique.
+ description:
+ type: string
+ nullable: true
+ description: Optional human-readable evaluator description.
+ type:
+ type: string
+ const: code
+ description: Evaluator type.
+ sourceCode:
+ type: string
+ description: Source code executed for each matched observation.
+ sourceCodeLanguage:
+ $ref: '#/components/schemas/CodeEvaluatorSourceCodeLanguage'
+ description: Runtime language used to execute the source code.
required:
- - data
- - meta
- CreateDatasetRunItemRequest:
- title: CreateDatasetRunItemRequest
+ - name
+ - type
+ - sourceCode
+ - sourceCodeLanguage
+ CreateEvaluatorRequest:
+ title: CreateEvaluatorRequest
+ oneOf:
+ - type: object
+ allOf:
+ - type: object
+ properties:
+ type:
+ type: string
+ enum:
+ - llm_as_judge
+ - $ref: '#/components/schemas/CreateLlmAsJudgeEvaluatorRequest'
+ required:
+ - type
+ - type: object
+ allOf:
+ - type: object
+ properties:
+ type:
+ type: string
+ enum:
+ - code
+ - $ref: '#/components/schemas/CreateCodeEvaluatorRequest'
+ required:
+ - type
+ description: >-
+ Initial evaluator definition with metadata and definition fields at the
+ same level. The returned evaluator starts at version `1`.
+ UpdateEvaluatorMetadataRequest:
+ title: UpdateEvaluatorMetadataRequest
type: object
+ description: Metadata-only update. At least one field must be provided.
properties:
- runName:
- type: string
- runDescription:
+ name:
type: string
nullable: true
- description: Description of the run. If run exists, description will be updated.
- metadata:
+ description: New human-readable evaluator name.
+ description:
+ type: string
nullable: true
- description: Metadata of the dataset run, updates run if run already exists
- datasetItemId:
+ description: New description. Set to `null` to clear it.
+ UpdateLlmAsJudgeEvaluatorRequest:
+ title: UpdateLlmAsJudgeEvaluatorRequest
+ type: object
+ properties:
+ name:
type: string
- observationId:
+ nullable: true
+ description: New human-readable evaluator name.
+ description:
type: string
nullable: true
- traceId:
+ description: New description. Set to `null` to clear it.
+ type:
type: string
+ const: llm_as_judge
+ description: Evaluator type. The type of an existing evaluator cannot change.
+ prompt:
+ $ref: '#/components/schemas/EvaluatorChatPromptInput'
+ description: >-
+ Complete replacement user prompt string or a list containing exactly
+ one user chat message.
+ modelConfig:
+ $ref: '#/components/schemas/EvaluatorModelConfig'
nullable: true
description: >-
- traceId should always be provided. For compatibility with older SDK
- versions it can also be inferred from the provided observationId.
- datasetVersion:
- type: string
- format: date-time
+ Explicit model configuration. Set to `null` or omit to use the
+ project's default evaluation model.
+ variableMapping:
+ type: array
+ items:
+ $ref: '#/components/schemas/PromptVariableMappingInput'
nullable: true
description: >-
- ISO 8601 timestamp (RFC 3339, Section 5.6) in UTC (e.g.,
- "2026-01-21T14:35:42Z").
-
- Specifies the dataset version to use for this experiment run.
-
- If provided, the experiment will use dataset items as they existed
- at or before this timestamp.
-
- If not provided, uses the latest version of dataset items.
- createdAt:
+ Complete replacement default variable mapping, or `null` when no
+ default is configured.
+ outputDefinition:
+ $ref: '#/components/schemas/EvaluatorOutputDefinition'
+ description: >-
+ Complete replacement output schema. Do not include an internal
+ persistence version.
+ required:
+ - type
+ - prompt
+ - outputDefinition
+ UpdateCodeEvaluatorRequest:
+ title: UpdateCodeEvaluatorRequest
+ type: object
+ properties:
+ name:
type: string
- format: date-time
nullable: true
- description: >-
- Optional timestamp to set the createdAt field of the dataset run
- item. If not provided or null, defaults to current timestamp.
+ description: New human-readable evaluator name.
+ description:
+ type: string
+ nullable: true
+ description: New description. Set to `null` to clear it.
+ type:
+ type: string
+ const: code
+ description: Evaluator type. The type of an existing evaluator cannot change.
+ sourceCode:
+ type: string
+ description: Complete replacement source code.
+ sourceCodeLanguage:
+ $ref: '#/components/schemas/CodeEvaluatorSourceCodeLanguage'
+ description: Runtime language used to execute the source code.
required:
- - runName
- - datasetItemId
- PaginatedDatasetRunItems:
- title: PaginatedDatasetRunItems
+ - type
+ - sourceCode
+ - sourceCodeLanguage
+ UpdateEvaluatorRequest:
+ title: UpdateEvaluatorRequest
+ oneOf:
+ - $ref: '#/components/schemas/UpdateEvaluatorMetadataRequest'
+ - $ref: '#/components/schemas/UpdateLlmAsJudgeEvaluatorRequest'
+ - $ref: '#/components/schemas/UpdateCodeEvaluatorRequest'
+ description: >-
+ Metadata-only update or complete flattened definition replacement. At
+ least one field must be provided.
+ EvaluatorsPage:
+ title: EvaluatorsPage
type: object
properties:
data:
type: array
items:
- $ref: '#/components/schemas/DatasetRunItem'
+ $ref: '#/components/schemas/Evaluator'
+ description: Evaluators for this page.
meta:
- $ref: '#/components/schemas/utilsMetaResponse'
- _deprecation:
- $ref: '#/components/schemas/Deprecation'
- nullable: true
+ $ref: '#/components/schemas/CursorMeta'
+ description: Cursor pagination metadata.
required:
- data
- meta
- PaginatedDatasets:
- title: PaginatedDatasets
+ EvaluatorVersionsPage:
+ title: EvaluatorVersionsPage
type: object
properties:
data:
type: array
items:
- $ref: '#/components/schemas/Dataset'
+ $ref: '#/components/schemas/EvaluatorVersion'
+ description: Evaluator versions for this page in newest-first order.
meta:
- $ref: '#/components/schemas/utilsMetaResponse'
+ $ref: '#/components/schemas/CursorMeta'
+ description: Cursor pagination metadata.
required:
- data
- meta
- CreateDatasetRequest:
- title: CreateDatasetRequest
+ CursorMeta:
+ title: CursorMeta
type: object
properties:
- name:
- type: string
- description:
+ cursor:
type: string
nullable: true
- metadata:
- nullable: true
- inputSchema:
- nullable: true
- description: >-
- JSON Schema for validating dataset item inputs. When set, all new
- and existing dataset items will be validated against this schema.
- expectedOutputSchema:
- nullable: true
- description: >-
- JSON Schema for validating dataset item expected outputs. When set,
- all new and existing dataset items will be validated against this
- schema.
- required:
- - name
- PaginatedDatasetRuns:
- title: PaginatedDatasetRuns
- type: object
- properties:
- data:
- type: array
- items:
- $ref: '#/components/schemas/DatasetRun'
- meta:
- $ref: '#/components/schemas/utilsMetaResponse'
- _deprecation:
- $ref: '#/components/schemas/Deprecation'
- nullable: true
- required:
- - data
- - meta
- DeleteDatasetRunResponse:
- title: DeleteDatasetRunResponse
+ description: Opaque cursor for the next page. Omitted when there is no next page.
+ DeletedEvaluator:
+ title: DeletedEvaluator
type: object
+ description: Confirmation returned after successful evaluator deletion.
properties:
- message:
+ id:
type: string
+ description: Identifier of the deleted evaluator.
required:
- - message
+ - id
ExperimentsResponse:
title: ExperimentsResponse
type: object
@@ -14481,7 +17186,7 @@ components:
- llm_as_judge
- code
description: |-
- The evaluator engine type.
+ The evaluator type.
The unstable public API supports LLM-as-a-judge and code evaluators.
unstableCodeEvaluatorSourceCodeLanguage:
@@ -14528,8 +17233,8 @@ components:
- `paused`: enabled, but Langfuse has blocked execution until the
underlying issue is resolved.
- unstableEvaluationRuleMappingSource:
- title: unstableEvaluationRuleMappingSource
+ unstablePromptVariableMappingSource:
+ title: unstablePromptVariableMappingSource
type: string
enum:
- input
@@ -15072,11 +17777,11 @@ components:
required:
- column
- operator
- unstableEvaluationRuleMapping:
- title: unstableEvaluationRuleMapping
+ unstablePromptVariableMappingInput:
+ title: unstablePromptVariableMappingInput
type: object
description: >-
- Maps one evaluator variable to one source field from the target object.
+ Connects one prompt variable to the available evaluation data.
Manual mappings are used for `llm_as_judge` evaluators. `code`
@@ -15107,10 +17812,10 @@ components:
- `invalid_variable_mapping`: the variable name is unknown for this
evaluator, or the selected `source` is not valid for the chosen `target`
- - `missing_variable_mapping`: one or more LLM-as-judge evaluator
- variables are not mapped yet
+ - `missing_variable_mapping`: one or more LLM-as-judge prompt variables
+ are not mapped yet
- - `duplicate_variable_mapping`: the same evaluator variable appears more
+ - `duplicate_variable_mapping`: the same prompt variable appears more
than once
- `invalid_json_path`: the JSONPath expression is malformed. Remove it
@@ -15125,7 +17830,7 @@ components:
Example: for the prompt `Judge {{input}} against {{output}}`, use
`input` and `output`.
source:
- $ref: '#/components/schemas/unstableEvaluationRuleMappingSource'
+ $ref: '#/components/schemas/unstablePromptVariableMappingSource'
description: >-
Source field that should populate the prompt variable.
@@ -15154,34 +17859,19 @@ components:
required:
- variable
- source
- unstableEvaluationRuleReadMapping:
- title: unstableEvaluationRuleReadMapping
+ unstablePromptVariableMappingRead:
+ title: unstablePromptVariableMappingRead
type: object
- description: >-
- Maps an evaluator variable to target data, as returned by read
- endpoints.
-
-
- Reads are more permissive than writes in two ways, so that data
- configured before a
-
- migration stays readable and repairable:
-
- - `source` is `null` when a mapping was never fully configured. Set a
- concrete `source` to fix it.
-
- - every `source` value is allowed regardless of the rule `target`,
- because an evaluator's
- default mapping is stored independently of the rules that use it.
+ description: |-
+ Connects one prompt variable to data from a target.
- Requests always require a concrete `source` that is valid for the
- selected `target`.
+ `source` is `null` when the mapping is incomplete.
properties:
variable:
type: string
description: Prompt variable name without braces.
source:
- $ref: '#/components/schemas/unstableEvaluationRuleMappingSource'
+ $ref: '#/components/schemas/unstablePromptVariableMappingSource'
nullable: true
description: >-
Source field populating the variable, or `null` when the mapping is
@@ -16198,7 +18888,7 @@ components:
- how often scoring should run
- - which target fields should populate each evaluator variable
+ - which target fields should populate each prompt variable
- whether the deployment is active, inactive, or paused
@@ -16303,7 +18993,7 @@ components:
mapping:
type: array
items:
- $ref: '#/components/schemas/unstableEvaluationRuleReadMapping'
+ $ref: '#/components/schemas/unstablePromptVariableMappingRead'
description: >-
Deprecated compatibility alias containing the effective mapping for
`evaluators[0]`.
@@ -16318,13 +19008,13 @@ components:
title: unstableLegacyEvaluationRule
type: object
description: >-
- Legacy trace- or dataset-level evaluation rule returned by list and get
- for migration.
+ **Deprecated:** Legacy trace- or dataset-level evaluation rule returned
+ by list and get for migration.
This resource is read-only through the unstable public API. Its mapping
preserves the trace, dataset item, or named observation selected for
- each evaluator variable.
+ each prompt variable.
properties:
evaluators:
type: array
@@ -16352,7 +19042,7 @@ components:
mapping:
type: array
items:
- $ref: '#/components/schemas/unstableLegacyEvaluationRuleMapping'
+ $ref: '#/components/schemas/unstableLegacyPromptVariableMapping'
description: >-
Stored variable mappings, including the trace, dataset item, or
named observation selected for each variable.
@@ -16412,7 +19102,7 @@ components:
`filter[].value`
- for `llm_as_judge`, fetch or inspect the evaluator first and provide a
- complete variable mapping for every evaluator variable
+ complete mapping for every prompt variable
- for `code`, do not send variables or mappings; Langfuse stores the
fixed code runtime mapping automatically
@@ -16508,15 +19198,12 @@ components:
mapping:
type: array
items:
- $ref: '#/components/schemas/unstableEvaluationRuleMapping'
- description: >-
+ $ref: '#/components/schemas/unstablePromptVariableMappingInput'
+ description: |-
LLM-as-judge variable mappings.
-
- Every evaluator variable must appear exactly once.
-
- Build this list from the evaluator `variables` array returned by the
- evaluator endpoints.
+ Every prompt variable must appear exactly once.
+ Build this list from the evaluator's `variables` array.
required:
- name
- evaluator
@@ -16599,7 +19286,7 @@ components:
assignment.
- If you change to an LLM-as-judge `evaluator`, send a fresh `mapping`
- unless you are certain the existing mapping still matches the evaluator
+ unless you are certain the existing mapping still matches the prompt
variables.
- If you change `target` for an LLM-as-judge rule, usually send both
@@ -16671,7 +19358,7 @@ components:
mapping:
type: array
items:
- $ref: '#/components/schemas/unstableEvaluationRuleMapping'
+ $ref: '#/components/schemas/unstablePromptVariableMappingInput'
nullable: true
description: >-
Updated LLM-as-judge variable mappings.
@@ -16722,7 +19409,7 @@ components:
type:
$ref: '#/components/schemas/unstableLlmAsJudgeEvaluatorType'
nullable: true
- description: Evaluator engine type. Defaults to `llm_as_judge` when omitted.
+ description: Evaluator type. Defaults to `llm_as_judge` when omitted.
required:
- name
unstableCodeEvaluationRuleEvaluatorReference:
@@ -16763,7 +19450,7 @@ components:
description: Evaluator family name.
type:
$ref: '#/components/schemas/unstableEvaluatorType'
- description: Evaluator engine type.
+ description: Evaluator type.
required:
- id
- name
@@ -16777,7 +19464,7 @@ components:
mapping:
type: array
items:
- $ref: '#/components/schemas/unstableEvaluationRuleReadMapping'
+ $ref: '#/components/schemas/unstablePromptVariableMappingRead'
nullable: true
description: >-
Rule-specific override, or `null` to inherit the evaluator version's
@@ -16788,13 +19475,14 @@ components:
unstableLegacyEvaluationRuleEvaluatorAssignment:
title: unstableLegacyEvaluationRuleEvaluatorAssignment
type: object
+ description: '**Deprecated:** Evaluator assignment on a legacy trace or dataset rule.'
properties:
evaluator:
$ref: '#/components/schemas/unstableEvaluationRuleEvaluator'
mapping:
type: array
items:
- $ref: '#/components/schemas/unstableLegacyEvaluationRuleMapping'
+ $ref: '#/components/schemas/unstableLegacyPromptVariableMapping'
nullable: true
required:
- evaluator
@@ -16811,12 +19499,13 @@ components:
enum:
- trace
- dataset
- unstableLegacyEvaluationRuleMapping:
- title: unstableLegacyEvaluationRuleMapping
+ description: '**Deprecated:** Legacy trace or dataset evaluation target.'
+ unstableLegacyPromptVariableMapping:
+ title: unstableLegacyPromptVariableMapping
type: object
description: >-
- Maps one evaluator variable to a trace, dataset item, or field on a
- named observation in a legacy rule.
+ **Deprecated:** Connects one prompt variable to data from a trace,
+ dataset item, or named observation in a legacy rule.
properties:
variable:
type: string
@@ -16854,6 +19543,9 @@ components:
- embedding
- guardrail
- dataset_item
+ description: >-
+ **Deprecated:** Legacy Langfuse object used by a trace or dataset
+ evaluation rule.
unstableCreateEvaluationRuleEvaluatorAssignment:
title: unstableCreateEvaluationRuleEvaluatorAssignment
type: object
@@ -16863,7 +19555,7 @@ components:
mapping:
type: array
items:
- $ref: '#/components/schemas/unstableEvaluationRuleMapping'
+ $ref: '#/components/schemas/unstablePromptVariableMappingInput'
nullable: true
description: >-
Optional rule-specific override. Omit to inherit the evaluator
@@ -16977,7 +19669,7 @@ components:
mapping:
type: array
items:
- $ref: '#/components/schemas/unstableEvaluationRuleReadMapping'
+ $ref: '#/components/schemas/unstablePromptVariableMappingRead'
nullable: true
description: >-
Default variable mapping for this evaluator version, or `null` when
@@ -17136,7 +19828,7 @@ components:
mapping:
type: array
items:
- $ref: '#/components/schemas/unstableEvaluationRuleMapping'
+ $ref: '#/components/schemas/unstablePromptVariableMappingInput'
nullable: true
description: >-
Optional default variable mapping inherited by rule assignments that
diff --git a/web/src/__tests__/server/evaluation-rule-v2-service.servertest.ts b/web/src/__tests__/server/evaluation-rule-v2-service.servertest.ts
index 0644077bb5..11961f013c 100644
--- a/web/src/__tests__/server/evaluation-rule-v2-service.servertest.ts
+++ b/web/src/__tests__/server/evaluation-rule-v2-service.servertest.ts
@@ -30,6 +30,7 @@ vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({
const orgIds: string[] = [];
let projectId = "";
let otherProjectId = "";
+let defaultEvaluatorId = "";
beforeAll(async () => {
const [first, second] = await Promise.all([
@@ -41,9 +42,10 @@ beforeAll(async () => {
orgIds.push(first.org.id, second.org.id);
});
-beforeEach(() => {
+beforeEach(async () => {
telemetryMocks.getRecentRuleExecutionTraces.mockReset();
telemetryMocks.getTotalCostByRule.mockReset();
+ defaultEvaluatorId = (await createEvaluator()).id;
});
afterEach(async () => {
@@ -115,7 +117,7 @@ function createLegacyRule(
});
}
-function createInput(evaluatorId?: string) {
+function createInput(evaluatorId: string | null = defaultEvaluatorId) {
return {
projectId,
name: "Production generations",
@@ -416,17 +418,12 @@ describe("RuleService", () => {
});
describe("create", () => {
- it("creates an active observation rule without evaluator assignments", async () => {
+ it("rejects an active observation rule without evaluator assignments", async () => {
await expect(
- createService().create(createInput(), null),
- ).resolves.toMatchObject({
- enabled: true,
- targetObject: "event",
- timeScope: ["NEW"],
- delay: 0,
- sampling: 0.25,
- assignments: [],
- });
+ createService().create(createInput(null), null),
+ ).rejects.toThrow(
+ "An enabled evaluation rule requires at least one evaluator assignment",
+ );
});
it("returns evaluator mapping fallback data", async () => {
@@ -721,7 +718,7 @@ describe("RuleService", () => {
const evaluator = await createEvaluator();
const service = createService();
const rule = await service.create(
- { ...createInput(), enabled: false },
+ { ...createInput(null), enabled: false },
null,
);
const attachment = {
@@ -1055,7 +1052,17 @@ describe("RuleService", () => {
});
await prisma.evaluationRule.update({
where: { id: legacyRule.id },
- data: { status: "ACTIVE" },
+ data: {
+ status: "ACTIVE",
+ filter: [
+ {
+ type: "string",
+ column: "Trace Name",
+ operator: "=",
+ value: "legacy trace",
+ },
+ ],
+ },
});
await service.update({
projectId,
@@ -1187,7 +1194,10 @@ describe("RuleService", () => {
undefined,
legacyMapping,
);
- const rule = await createService().create(createInput(), null);
+ const rule = await createService().create(
+ { ...createInput(null), enabled: false },
+ null,
+ );
const updated = await createService().attach({
projectId,
@@ -1223,18 +1233,63 @@ describe("RuleService", () => {
},
]);
});
+
+ it("rejects enabling a rule whose resulting assignment list is empty", async () => {
+ const service = createService();
+ const evaluator = await createEvaluator();
+ const assigned = await service.create(createInput(evaluator.id), null);
+ const unassigned = await service.create(
+ { ...createInput(null), enabled: false },
+ null,
+ );
+
+ await expect(
+ service.update({
+ projectId,
+ ruleId: assigned.id,
+ enabled: true,
+ evaluatorMappings: [],
+ }),
+ ).rejects.toThrow(
+ "An enabled evaluation rule requires at least one evaluator assignment",
+ );
+ await expect(
+ service.update({
+ projectId,
+ ruleId: unassigned.id,
+ enabled: true,
+ }),
+ ).rejects.toThrow(
+ "An enabled evaluation rule requires at least one evaluator assignment",
+ );
+ });
});
describe("setEnabled", () => {
it("updates the rule status", async () => {
+ const evaluator = await createEvaluator();
const service = createService();
- const rule = await service.create(createInput(), null);
+ const rule = await service.create(createInput(evaluator.id), null);
await expect(
service.setEnabled({ projectId, ruleId: rule.id, enabled: false }),
).resolves.toMatchObject({ id: rule.id, enabled: false });
});
+ it("rejects enabling a rule without evaluator assignments", async () => {
+ const service = createService();
+ const rule = await service.create(
+ { ...createInput(null), enabled: false },
+ null,
+ );
+
+ await expect(
+ service.setEnabled({ projectId, ruleId: rule.id, enabled: true }),
+ ).rejects.toThrow(
+ "An enabled evaluation rule requires at least one evaluator assignment",
+ );
+ });
+
it("rejects an unavailable rule", async () => {
await expect(
createService().setEnabled({
@@ -1325,6 +1380,31 @@ describe("RuleService", () => {
});
describe("setManyEnabled", () => {
+ it("rejects enabling any selected rule without evaluator assignments", async () => {
+ const service = createService();
+ const [first, second] = await Promise.all([
+ service.create({ ...createInput(null), enabled: false }, null),
+ service.create(
+ {
+ ...createInput(null),
+ name: "Second unassigned rule",
+ enabled: false,
+ },
+ null,
+ ),
+ ]);
+
+ await expect(
+ service.setManyEnabled({
+ projectId,
+ ruleIds: [first.id, second.id],
+ enabled: true,
+ }),
+ ).rejects.toThrow(
+ "An enabled evaluation rule requires at least one evaluator assignment",
+ );
+ });
+
it("updates filtered matches without crossing project boundaries", async () => {
const service = createService();
const first = await service.create(createInput(), null);
@@ -1332,9 +1412,10 @@ describe("RuleService", () => {
{ ...createInput(), name: "Other production rule" },
null,
);
+ const foreignEvaluator = await createEvaluator(otherProjectId);
await createService().create(
{
- ...createInput(),
+ ...createInput(foreignEvaluator.id),
projectId: otherProjectId,
name: "Foreign production rule",
},
@@ -1345,13 +1426,24 @@ describe("RuleService", () => {
projectId,
isBatchAction: true,
search: "production",
+ filter: [
+ {
+ type: "string",
+ column: "name",
+ operator: "=",
+ value: first.name,
+ },
+ ],
enabled: false,
});
- expect(new Set(changedIds)).toEqual(new Set([first.id, second.id]));
+ expect(changedIds).toEqual([first.id]);
await expect(
service.list({ projectId, page: 1, limit: 50, enabled: false }),
- ).resolves.toMatchObject({ totalItems: 2 });
+ ).resolves.toMatchObject({ totalItems: 1 });
+ await expect(service.get(projectId, second.id)).resolves.toMatchObject({
+ enabled: true,
+ });
});
});
@@ -1381,6 +1473,13 @@ describe("RuleService", () => {
{ ...createInput(), name: "Second rule" },
null,
);
+ await prisma.jobExecution.createMany({
+ data: [first.id, second.id].map((jobConfigurationId) => ({
+ projectId,
+ jobConfigurationId,
+ status: "PENDING" as const,
+ })),
+ });
await expect(
service.deleteMany({ projectId, ruleIds: [first.id, second.id] }),
@@ -1388,14 +1487,23 @@ describe("RuleService", () => {
await expect(
service.list({ projectId, page: 1, limit: 50 }),
).resolves.toEqual({ rules: [], totalItems: 0 });
+ await expect(
+ prisma.jobExecution.count({
+ where: {
+ projectId,
+ jobConfigurationId: { in: [first.id, second.id] },
+ },
+ }),
+ ).resolves.toBe(0);
});
it("ignores unavailable rules in an explicit selection", async () => {
const service = createService();
const local = await service.create(createInput(), null);
+ const foreignEvaluator = await createEvaluator(otherProjectId);
const foreign = await createService().create(
{
- ...createInput(),
+ ...createInput(foreignEvaluator.id),
projectId: otherProjectId,
name: "Foreign rule",
},
@@ -1456,10 +1564,13 @@ describe("RuleService", () => {
},
],
});
- await service.detach({
+ await service.attach({
projectId,
ruleId: rule.id,
- evaluatorId: secondEvaluator.id,
+ assignment: {
+ evaluatorId: firstEvaluator.id,
+ variableMapping: null,
+ },
});
await service.setManyEnabled({
projectId,
diff --git a/web/src/__tests__/server/evaluation-rule-v2-trpc.servertest.ts b/web/src/__tests__/server/evaluation-rule-v2-trpc.servertest.ts
index b2f6ac1c90..74b9a0a9cc 100644
--- a/web/src/__tests__/server/evaluation-rule-v2-trpc.servertest.ts
+++ b/web/src/__tests__/server/evaluation-rule-v2-trpc.servertest.ts
@@ -125,7 +125,8 @@ function ruleInput(evaluatorId?: string) {
describe("evaluation rule v2 tRPC", () => {
it("supports the rule lifecycle", async () => {
- const created = await caller.evalsV2.rules.create(ruleInput());
+ const evaluator = await createEvaluator("Transport evaluator");
+ const created = await caller.evalsV2.rules.create(ruleInput(evaluator.id));
const updated = await caller.evalsV2.rules.update({
projectId,
ruleId: created.id,
diff --git a/web/src/__tests__/server/evaluation-rules-public-api.servertest.ts b/web/src/__tests__/server/evaluation-rules-public-api.servertest.ts
new file mode 100644
index 0000000000..4ccf22e21f
--- /dev/null
+++ b/web/src/__tests__/server/evaluation-rules-public-api.servertest.ts
@@ -0,0 +1,613 @@
+import { randomUUID } from "node:crypto";
+import { EvalTargetObject, EvalTemplateType } from "@langfuse/shared";
+import { prisma } from "@langfuse/shared/src/db";
+import { createOrgProjectAndApiKey } from "@langfuse/shared/src/server";
+import {
+ makeAPICall,
+ makeZodVerifiedAPICall,
+} from "@/src/__tests__/test-utils";
+import {
+ createCodeEvaluator,
+ createEvaluationRule,
+} from "@/src/__tests__/server/stable-evaluation-public-api-test-utils";
+import { EvaluatorService, RuleService } from "@/src/features/evals/server";
+import {
+ DeleteEvaluationRuleResponse,
+ EvaluationRule,
+ Evaluator,
+ ListEvaluationRulesResponse,
+ PublicApiError,
+} from "@/src/features/public-api";
+
+const createLegacyRule = (projectId: string, name: string) =>
+ prisma.evaluationRule.create({
+ data: {
+ projectId,
+ name,
+ targetObject: EvalTargetObject.TRACE,
+ status: "INACTIVE",
+ filter: [
+ {
+ type: "string",
+ column: "Trace Name",
+ key: "legacy-filter-key",
+ operator: "=",
+ value: "legacy trace",
+ },
+ ],
+ sampling: 1,
+ delay: 0,
+ timeScope: ["NEW"],
+ },
+ });
+
+describe("stable evaluation rules public API", () => {
+ it("creates evaluation rules", async () => {
+ const { auth } = await createOrgProjectAndApiKey();
+ const evaluator = await createCodeEvaluator("rule evaluator", auth);
+ const datasetId = randomUUID();
+ const body = {
+ name: "same rule name",
+ enabled: false,
+ filter: [
+ {
+ type: "boolean" as const,
+ column: "isExperimentItemRootSpan" as const,
+ operator: "=" as const,
+ value: true,
+ },
+ {
+ type: "stringOptions" as const,
+ column: "datasetId" as const,
+ operator: "any of" as const,
+ value: [datasetId],
+ },
+ {
+ type: "null" as const,
+ column: "parentObservationId" as const,
+ operator: "is null" as const,
+ value: "" as const,
+ },
+ ],
+ evaluatorAssignments: [
+ { evaluatorId: evaluator.body.id, variableMapping: null },
+ ],
+ };
+
+ const first = await makeZodVerifiedAPICall(
+ EvaluationRule,
+ "POST",
+ "/api/public/v2/evaluation-rules",
+ body,
+ auth,
+ 201,
+ );
+ const second = await makeZodVerifiedAPICall(
+ EvaluationRule,
+ "POST",
+ "/api/public/v2/evaluation-rules",
+ body,
+ auth,
+ 201,
+ );
+ expect(first.body.id).not.toBe(second.body.id);
+ expect(first.body).toMatchObject({
+ createdBy: null,
+ sampling: 1,
+ evaluatorAssignments: [
+ {
+ evaluatorId: evaluator.body.id,
+ variableMapping: null,
+ },
+ ],
+ });
+ expect(first.body).not.toHaveProperty("target");
+ expect(first.body.filter).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ column: "isExperimentItemRootSpan",
+ value: true,
+ }),
+ expect.objectContaining({
+ column: "datasetId",
+ value: [datasetId],
+ }),
+ expect.objectContaining({
+ type: "null",
+ column: "parentObservationId",
+ operator: "is null",
+ value: "",
+ }),
+ ]),
+ );
+
+ const nullSampling = await makeAPICall(
+ "POST",
+ "/api/public/v2/evaluation-rules",
+ {
+ ...body,
+ name: "invalid nullable sampling",
+ sampling: null,
+ },
+ auth,
+ );
+ expect(nullSampling.status).toBe(400);
+
+ const enabledWithoutEvaluator = await makeAPICall(
+ "POST",
+ "/api/public/v2/evaluation-rules",
+ {
+ name: "invalid active draft",
+ enabled: true,
+ sampling: 1,
+ filter: [],
+ evaluatorAssignments: [],
+ },
+ auth,
+ );
+ expect(enabledWithoutEvaluator.status).toBe(400);
+ });
+
+ it("gets an evaluation rule", async () => {
+ const { auth, projectId } = await createOrgProjectAndApiKey();
+ const user = await prisma.user.create({
+ data: {
+ id: randomUUID(),
+ name: "Rule Author",
+ email: `${randomUUID()}@example.com`,
+ },
+ });
+ const evaluator = await new EvaluatorService(
+ prisma,
+ async () => undefined,
+ ).create(
+ {
+ projectId,
+ name: "rule author evaluator",
+ description: null,
+ definition: {
+ type: EvalTemplateType.CODE,
+ sourceCode: "function evaluate() { return { scores: [] }; }",
+ sourceCodeLanguage: "TYPESCRIPT",
+ },
+ },
+ user.id,
+ );
+ const rule = await new RuleService(prisma, async () => undefined).create(
+ {
+ projectId,
+ name: "authored rule",
+ targetObject: EvalTargetObject.EVENT,
+ enabled: false,
+ sampling: 1,
+ filter: [],
+ evaluatorAssignments: [
+ { evaluatorId: evaluator.id, variableMapping: null },
+ ],
+ },
+ user.id,
+ );
+
+ const response = await makeZodVerifiedAPICall(
+ EvaluationRule,
+ "GET",
+ `/api/public/v2/evaluation-rules/${rule.id}`,
+ undefined,
+ auth,
+ );
+ expect(response.body.createdBy).toEqual({
+ id: user.id,
+ name: user.name,
+ });
+
+ const legacy = await createLegacyRule(projectId, "legacy get rule");
+ await prisma.evaluationRule.update({
+ where: { id: legacy.id },
+ data: {
+ filter: [
+ {
+ type: "string",
+ column: "Trace Name",
+ operator: "=",
+ value: "legacy trace",
+ },
+ {
+ type: "booleanObject",
+ column: "metadata",
+ key: "reviewed",
+ operator: "=",
+ value: true,
+ },
+ {
+ type: "positionInTrace",
+ column: "positionInTrace",
+ operator: "=",
+ key: "root",
+ },
+ ],
+ },
+ });
+ await prisma.evaluationRuleEvaluatorAssignment.create({
+ data: {
+ projectId,
+ evaluationRuleId: legacy.id,
+ evaluatorId: evaluator.id,
+ variableMapping: [
+ {
+ templateVariable: "input",
+ langfuseObject: "trace",
+ objectName: null,
+ selectedColumnId: "input",
+ jsonSelector: null,
+ },
+ {
+ templateVariable: "output",
+ langfuseObject: "generation",
+ objectName: "answer-generation",
+ selectedColumnId: "output",
+ jsonSelector: "$.answer",
+ },
+ ],
+ },
+ });
+ const legacyResponse = await makeZodVerifiedAPICall(
+ EvaluationRule,
+ "GET",
+ `/api/public/v2/evaluation-rules/${legacy.id}`,
+ undefined,
+ auth,
+ );
+ expect(legacyResponse.body).toMatchObject({
+ id: legacy.id,
+ filter: [
+ { column: "Trace Name", value: "legacy trace" },
+ {
+ type: "booleanObject",
+ column: "metadata",
+ key: "reviewed",
+ value: true,
+ },
+ {
+ type: "positionInTrace",
+ column: "positionInTrace",
+ key: "root",
+ },
+ ],
+ evaluatorAssignments: [
+ {
+ variableMapping: [
+ {
+ mappingType: "legacy",
+ variable: "input",
+ langfuseObject: "trace",
+ objectName: null,
+ source: "input",
+ },
+ {
+ mappingType: "legacy",
+ variable: "output",
+ langfuseObject: "generation",
+ objectName: "answer-generation",
+ source: "output",
+ jsonPath: "$.answer",
+ },
+ ],
+ },
+ ],
+ });
+ });
+
+ it("returns 404 when getting an invalid evaluation rule ID", async () => {
+ const { auth } = await createOrgProjectAndApiKey();
+ const missing = await makeAPICall(
+ "GET",
+ `/api/public/v2/evaluation-rules/${randomUUID()}`,
+ undefined,
+ auth,
+ );
+ expect(missing.status).toBe(404);
+ });
+
+ it("rejects an LLM evaluator assignment when no variable mapping is configured", async () => {
+ const { auth } = await createOrgProjectAndApiKey();
+ const evaluator = await makeZodVerifiedAPICall(
+ Evaluator,
+ "POST",
+ "/api/public/v2/evaluators",
+ {
+ name: "LLM evaluator without default mapping",
+ type: "llm_as_judge",
+ prompt: [{ role: "user", content: "Judge {{input}}" }],
+ outputDefinition: {
+ dataType: "NUMERIC",
+ minValue: 0,
+ maxValue: 1,
+ },
+ },
+ auth,
+ 201,
+ );
+
+ const rule = await makeAPICall(
+ "POST",
+ "/api/public/v2/evaluation-rules",
+ {
+ name: "Rule without variable mapping",
+ enabled: false,
+ evaluatorAssignments: [{ evaluatorId: evaluator.body.id }],
+ },
+ auth,
+ );
+
+ expect(rule.status).toBe(400);
+ expect(PublicApiError.parse(rule.body)).toMatchObject({
+ code: "invalid_request",
+ message: "Missing mappings for evaluator variables: input",
+ });
+ });
+
+ it("lists evaluation rules", async () => {
+ const { auth, projectId } = await createOrgProjectAndApiKey();
+ const evaluator = await createCodeEvaluator("listed rule evaluator", auth);
+ const modern = await createEvaluationRule({
+ name: "listed modern rule",
+ evaluatorId: evaluator.body.id,
+ auth,
+ });
+ const legacy = await createLegacyRule(projectId, "legacy listed rule");
+
+ const response = await makeZodVerifiedAPICall(
+ ListEvaluationRulesResponse,
+ "GET",
+ "/api/public/v2/evaluation-rules",
+ undefined,
+ auth,
+ );
+ expect(response.body.data).toContainEqual(
+ expect.objectContaining({ id: modern.body.id }),
+ );
+ expect(response.body.data.find(({ id }) => id === legacy.id)).toMatchObject(
+ {
+ id: legacy.id,
+ filter: [
+ {
+ type: "string",
+ column: "Trace Name",
+ key: "legacy-filter-key",
+ operator: "=",
+ value: "legacy trace",
+ },
+ ],
+ },
+ );
+ });
+
+ it("lists evaluation rules with cursor pagination", async () => {
+ const { auth } = await createOrgProjectAndApiKey();
+ const evaluator = await createCodeEvaluator(
+ "paginated rule evaluator",
+ auth,
+ );
+ const created = await Promise.all([
+ createEvaluationRule({
+ name: "paginated rule one",
+ evaluatorId: evaluator.body.id,
+ auth,
+ }),
+ createEvaluationRule({
+ name: "paginated rule two",
+ evaluatorId: evaluator.body.id,
+ auth,
+ }),
+ ]);
+
+ const firstPage = await makeZodVerifiedAPICall(
+ ListEvaluationRulesResponse,
+ "GET",
+ "/api/public/v2/evaluation-rules?limit=1",
+ undefined,
+ auth,
+ );
+ expect(firstPage.body.data).toHaveLength(1);
+ expect(firstPage.body.meta.cursor).toEqual(expect.any(String));
+
+ const secondPage = await makeZodVerifiedAPICall(
+ ListEvaluationRulesResponse,
+ "GET",
+ `/api/public/v2/evaluation-rules?limit=1&cursor=${encodeURIComponent(firstPage.body.meta.cursor!)}`,
+ undefined,
+ auth,
+ );
+ expect(secondPage.body.data).toHaveLength(1);
+ expect(secondPage.body.meta.cursor).toBeUndefined();
+ expect(
+ new Set(
+ [...firstPage.body.data, ...secondPage.body.data].map(({ id }) => id),
+ ),
+ ).toEqual(new Set(created.map(({ body: { id } }) => id)));
+ });
+
+ it("patches an evaluation rule", async () => {
+ const { auth, projectId } = await createOrgProjectAndApiKey();
+ const evaluator = await createCodeEvaluator("patched rule evaluator", auth);
+ const rule = await createEvaluationRule({
+ name: "rule before patch",
+ evaluatorId: evaluator.body.id,
+ auth,
+ });
+
+ const response = await makeZodVerifiedAPICall(
+ EvaluationRule,
+ "PATCH",
+ `/api/public/v2/evaluation-rules/${rule.body.id}`,
+ { name: "rule after patch", sampling: 0.5 },
+ auth,
+ );
+ expect(response.body).toMatchObject({
+ id: rule.body.id,
+ name: "rule after patch",
+ sampling: 0.5,
+ });
+
+ const contradictory = await makeAPICall(
+ "PATCH",
+ `/api/public/v2/evaluation-rules/${rule.body.id}`,
+ { enabled: true, evaluatorAssignments: [] },
+ auth,
+ );
+ expect(contradictory.status).toBe(400);
+ expect(PublicApiError.parse(contradictory.body)).toMatchObject({
+ code: "invalid_request",
+ message:
+ "An enabled evaluation rule requires at least one evaluator assignment",
+ });
+
+ const detached = await makeZodVerifiedAPICall(
+ EvaluationRule,
+ "PATCH",
+ `/api/public/v2/evaluation-rules/${rule.body.id}`,
+ { evaluatorAssignments: [] },
+ auth,
+ );
+ expect(detached.body).toMatchObject({
+ enabled: false,
+ evaluatorAssignments: [],
+ });
+
+ const enableWithoutAssignments = await makeAPICall(
+ "PATCH",
+ `/api/public/v2/evaluation-rules/${rule.body.id}`,
+ { enabled: true },
+ auth,
+ );
+ expect(enableWithoutAssignments.status).toBe(400);
+
+ const storedExperimentRule = await prisma.evaluationRule.create({
+ data: {
+ projectId,
+ name: "stored experiment rule",
+ targetObject: EvalTargetObject.EVENT,
+ status: "INACTIVE",
+ filter: [
+ {
+ type: "boolean",
+ column: "isExperimentItemRootSpan",
+ operator: "=",
+ value: true,
+ },
+ ],
+ sampling: 1,
+ delay: 0,
+ timeScope: ["NEW"],
+ },
+ });
+ const experimentResponse = await makeZodVerifiedAPICall(
+ EvaluationRule,
+ "PATCH",
+ `/api/public/v2/evaluation-rules/${storedExperimentRule.id}`,
+ { filter: [] },
+ auth,
+ );
+ expect(experimentResponse.body.filter).toEqual([]);
+ await expect(
+ prisma.evaluationRule.findUnique({
+ where: { id: storedExperimentRule.id },
+ select: { targetObject: true, filter: true },
+ }),
+ ).resolves.toMatchObject({
+ targetObject: EvalTargetObject.EVENT,
+ filter: [],
+ });
+ });
+
+ it("matches the existing legacy rule update restrictions", async () => {
+ const { auth, projectId } = await createOrgProjectAndApiKey();
+ const legacy = await createLegacyRule(projectId, "legacy patch rule");
+
+ const missing = await makeAPICall(
+ "PATCH",
+ `/api/public/v2/evaluation-rules/${randomUUID()}`,
+ { name: "missing rule" },
+ auth,
+ );
+ expect(missing.status).toBe(404);
+
+ const legacyResponse = await makeZodVerifiedAPICall(
+ EvaluationRule,
+ "PATCH",
+ `/api/public/v2/evaluation-rules/${legacy.id}`,
+ { enabled: false },
+ auth,
+ );
+ expect(legacyResponse.body).toMatchObject({
+ id: legacy.id,
+ enabled: false,
+ });
+
+ const unsupportedUpdate = await makeAPICall(
+ "PATCH",
+ `/api/public/v2/evaluation-rules/${legacy.id}`,
+ { name: "mutated legacy rule" },
+ auth,
+ );
+ expect(unsupportedUpdate.status).toBe(400);
+ await expect(
+ prisma.evaluationRule.findUnique({ where: { id: legacy.id } }),
+ ).resolves.toMatchObject({
+ name: "legacy patch rule",
+ status: "INACTIVE",
+ targetObject: EvalTargetObject.TRACE,
+ });
+ });
+
+ it("deletes an evaluation rule", async () => {
+ const { auth } = await createOrgProjectAndApiKey();
+ const evaluator = await createCodeEvaluator("deleted rule evaluator", auth);
+ const rule = await createEvaluationRule({
+ name: "deleted rule",
+ evaluatorId: evaluator.body.id,
+ auth,
+ });
+
+ const response = await makeZodVerifiedAPICall(
+ DeleteEvaluationRuleResponse,
+ "DELETE",
+ `/api/public/v2/evaluation-rules/${rule.body.id}`,
+ undefined,
+ auth,
+ );
+ expect(response.body.id).toBe(rule.body.id);
+
+ const get = await makeAPICall(
+ "GET",
+ `/api/public/v2/evaluation-rules/${rule.body.id}`,
+ undefined,
+ auth,
+ );
+ expect(get.status).toBe(404);
+ });
+
+ it("deletes legacy rules and returns 404 for missing rules", async () => {
+ const { auth, projectId } = await createOrgProjectAndApiKey();
+ const legacy = await createLegacyRule(projectId, "legacy delete rule");
+ const missing = await makeAPICall(
+ "DELETE",
+ `/api/public/v2/evaluation-rules/${randomUUID()}`,
+ undefined,
+ auth,
+ );
+ expect(missing.status).toBe(404);
+
+ const legacyResponse = await makeZodVerifiedAPICall(
+ DeleteEvaluationRuleResponse,
+ "DELETE",
+ `/api/public/v2/evaluation-rules/${legacy.id}`,
+ undefined,
+ auth,
+ );
+ expect(legacyResponse.body.id).toBe(legacy.id);
+ await expect(
+ prisma.evaluationRule.findUnique({ where: { id: legacy.id } }),
+ ).resolves.toBeNull();
+ });
+});
diff --git a/web/src/__tests__/server/evaluator-history-public-api.servertest.ts b/web/src/__tests__/server/evaluator-history-public-api.servertest.ts
new file mode 100644
index 0000000000..15f1f213cf
--- /dev/null
+++ b/web/src/__tests__/server/evaluator-history-public-api.servertest.ts
@@ -0,0 +1,104 @@
+import { randomUUID } from "node:crypto";
+import { createOrgProjectAndApiKey } from "@langfuse/shared/src/server";
+import {
+ makeAPICall,
+ makeZodVerifiedAPICall,
+} from "@/src/__tests__/test-utils";
+import {
+ codeDefinition,
+ createCodeEvaluator,
+} from "@/src/__tests__/server/stable-evaluation-public-api-test-utils";
+import {
+ Evaluator,
+ ListEvaluatorVersionsResponse,
+} from "@/src/features/public-api";
+
+describe("stable evaluator history public API", () => {
+ it("lists evaluator version history", async () => {
+ const { auth } = await createOrgProjectAndApiKey();
+ const evaluator = await createCodeEvaluator("history evaluator", auth);
+ const changedSource =
+ "function evaluate() { return { scores: [{ name: 'ok', value: true }] }; }";
+ await makeZodVerifiedAPICall(
+ Evaluator,
+ "PATCH",
+ `/api/public/v2/evaluators/${evaluator.body.id}`,
+ codeDefinition(changedSource),
+ auth,
+ );
+
+ const response = await makeZodVerifiedAPICall(
+ ListEvaluatorVersionsResponse,
+ "GET",
+ `/api/public/v2/evaluators/${evaluator.body.id}/versions`,
+ undefined,
+ auth,
+ );
+ expect(response.body.data).toHaveLength(2);
+ expect(response.body.data.map(({ version }) => version)).toEqual([2, 1]);
+ expect(response.body.data[0]).toMatchObject({
+ type: "code",
+ version: 2,
+ sourceCode: changedSource,
+ createdBy: null,
+ });
+ expect(response.body.meta.cursor).toBeUndefined();
+ });
+
+ it("returns 404 for an invalid evaluator ID with or without a cursor", async () => {
+ const { auth } = await createOrgProjectAndApiKey();
+ const evaluatorId = randomUUID();
+ const cursor = Buffer.from(JSON.stringify({ v: 1, version: 1 })).toString(
+ "base64url",
+ );
+
+ for (const query of ["", `?cursor=${encodeURIComponent(cursor)}`]) {
+ const response = await makeAPICall(
+ "GET",
+ `/api/public/v2/evaluators/${evaluatorId}/versions${query}`,
+ undefined,
+ auth,
+ );
+ expect(response.status).toBe(404);
+ }
+ });
+
+ it("lists evaluator version history with cursor pagination", async () => {
+ const { auth } = await createOrgProjectAndApiKey();
+ const evaluator = await createCodeEvaluator(
+ "paginated history evaluator",
+ auth,
+ );
+ await makeZodVerifiedAPICall(
+ Evaluator,
+ "PATCH",
+ `/api/public/v2/evaluators/${evaluator.body.id}`,
+ codeDefinition(
+ "function evaluate() { return { scores: [{ name: 'ok', value: true }] }; }",
+ ),
+ auth,
+ );
+
+ const firstPage = await makeZodVerifiedAPICall(
+ ListEvaluatorVersionsResponse,
+ "GET",
+ `/api/public/v2/evaluators/${evaluator.body.id}/versions?limit=1`,
+ undefined,
+ auth,
+ );
+ expect(firstPage.body.data).toHaveLength(1);
+ expect(firstPage.body.data[0]?.version).toBe(2);
+ expect(firstPage.body.meta.cursor).toEqual(expect.any(String));
+
+ const secondPage = await makeZodVerifiedAPICall(
+ ListEvaluatorVersionsResponse,
+ "GET",
+ `/api/public/v2/evaluators/${evaluator.body.id}/versions?limit=1&cursor=${encodeURIComponent(firstPage.body.meta.cursor!)}`,
+ undefined,
+ auth,
+ );
+ expect(secondPage.body.data).toHaveLength(1);
+ expect(secondPage.body.data[0]?.version).toBe(1);
+ expect(secondPage.body.meta.cursor).toBeUndefined();
+ });
+});
diff --git a/web/src/__tests__/server/evaluator-v2-repository.servertest.ts b/web/src/__tests__/server/evaluator-v2-repository.servertest.ts
index 1f5ef1b294..d9a0255a1f 100644
--- a/web/src/__tests__/server/evaluator-v2-repository.servertest.ts
+++ b/web/src/__tests__/server/evaluator-v2-repository.servertest.ts
@@ -38,7 +38,6 @@ const llmDefinition = (
vars: ["output"],
variableMapping: null,
outputDefinition: {
- version: 2,
dataType: "NUMERIC",
score: { description: "Quality" },
reasoning: { description: "Reasoning" },
diff --git a/web/src/__tests__/server/evaluator-v2-service.servertest.ts b/web/src/__tests__/server/evaluator-v2-service.servertest.ts
index 3715e25e64..f8d32f5275 100644
--- a/web/src/__tests__/server/evaluator-v2-service.servertest.ts
+++ b/web/src/__tests__/server/evaluator-v2-service.servertest.ts
@@ -37,6 +37,7 @@ import {
const mocks = vi.hoisted(() => ({
generateLangfuseAIText: vi.fn(),
getRecentEvaluatorExecutionTraces: vi.fn(),
+ invalidateProjectEvalConfigCaches: vi.fn(),
assertEvaluatorConfigurationValid: vi.fn(),
getEvaluatorDefinitionPreflightError: vi.fn(),
testEvaluator: vi.fn(),
@@ -53,6 +54,7 @@ vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({
...(await importOriginal<typeof SharedServer>()),
generateLangfuseAIText: mocks.generateLangfuseAIText,
getRecentEvaluatorExecutionTraces: mocks.getRecentEvaluatorExecutionTraces,
+ invalidateProjectEvalConfigCaches: mocks.invalidateProjectEvalConfigCaches,
}));
vi.mock("@/src/env.mjs", async (importOriginal) => {
@@ -114,7 +116,6 @@ const llmInput = (
{ templateVariable: "output", selectedColumnId: "output" },
],
outputDefinition: {
- version: 2,
dataType: "NUMERIC",
score: { description: "Quality" },
reasoning: { description: "Reasoning" },
@@ -163,6 +164,7 @@ beforeAll(async () => {
beforeEach(() => {
mocks.generateLangfuseAIText.mockReset();
mocks.getRecentEvaluatorExecutionTraces.mockReset();
+ mocks.invalidateProjectEvalConfigCaches.mockReset();
mocks.assertEvaluatorConfigurationValid.mockReset();
mocks.assertEvaluatorConfigurationValid.mockResolvedValue(undefined);
mocks.getEvaluatorDefinitionPreflightError.mockReset();
@@ -490,6 +492,84 @@ describe("EvaluatorService", () => {
"Evaluator paused: no valid evaluation model is configured. Update the evaluator template or default evaluation model, then reactivate it.",
versions: [expect.objectContaining({ version: 2 })],
});
+
+ mocks.invalidateProjectEvalConfigCaches.mockClear();
+ const recovered = await service.patch(
+ {
+ projectId,
+ evaluatorId: evaluator.id,
+ definition: {
+ ...llmInput("Invalid model").definition,
+ provider: "openai",
+ model: "available-model",
+ },
+ },
+ null,
+ );
+ expect(recovered).toMatchObject({
+ id: evaluator.id,
+ blockedAt: null,
+ blockReason: null,
+ blockMessage: null,
+ versions: [expect.objectContaining({ version: 3 })],
+ });
+ expect(mocks.invalidateProjectEvalConfigCaches).toHaveBeenCalledWith(
+ projectId,
+ );
+
+ await prisma.evaluator.update({
+ where: { id: evaluator.id },
+ data: {
+ blockedAt: new Date(),
+ blockReason: "LLM_CONNECTION_AUTH_INVALID",
+ blockMessage: "Authentication failed",
+ },
+ });
+ const stillBlocked = await service.patch(
+ {
+ projectId,
+ evaluatorId: evaluator.id,
+ definition: {
+ ...llmInput("Invalid model").definition,
+ prompt: "Updated prompt: {{output}}",
+ provider: "openai",
+ model: "available-model",
+ },
+ },
+ null,
+ );
+ expect(stillBlocked).toMatchObject({
+ blockedAt: expect.any(Date),
+ blockReason: "LLM_CONNECTION_AUTH_INVALID",
+ blockMessage: "Authentication failed",
+ });
+
+ await prisma.evaluator.update({
+ where: { id: evaluator.id },
+ data: {
+ blockedAt: new Date(),
+ blockReason: null,
+ blockMessage: "Legacy pause without a reason",
+ },
+ });
+ const legacyPause = await service.patch(
+ {
+ projectId,
+ evaluatorId: evaluator.id,
+ definition: {
+ ...llmInput("Invalid model").definition,
+ prompt: "Another updated prompt: {{output}}",
+ provider: "openai",
+ model: "available-model",
+ },
+ },
+ null,
+ );
+ expect(legacyPause).toMatchObject({
+ blockedAt: expect.any(Date),
+ blockReason: null,
+ blockMessage: "Legacy pause without a reason",
+ });
});
it("reactivates a blocked evaluator only after its model test succeeds", async () => {
diff --git a/web/src/__tests__/server/evaluator-v2-trpc.servertest.ts b/web/src/__tests__/server/evaluator-v2-trpc.servertest.ts
index 961e716d9a..9d85dfb987 100644
--- a/web/src/__tests__/server/evaluator-v2-trpc.servertest.ts
+++ b/web/src/__tests__/server/evaluator-v2-trpc.servertest.ts
@@ -34,13 +34,9 @@ let session: Session & { user: NonNullable<Session["user"]> };
const definition = {
type: "LLM_AS_JUDGE" as const,
prompt: "Judge {{output}}",
- provider: null,
- model: null,
- modelParams: null,
- vars: ["output"],
+ modelConfig: null,
variableMapping: [{ templateVariable: "output", selectedColumnId: "output" }],
outputDefinition: {
- version: 2 as const,
dataType: "NUMERIC" as const,
score: { description: "Quality" },
reasoning: { description: "Reasoning" },
diff --git a/web/src/__tests__/server/evaluators-public-api.servertest.ts b/web/src/__tests__/server/evaluators-public-api.servertest.ts
new file mode 100644
index 0000000000..ef27d3601c
--- /dev/null
+++ b/web/src/__tests__/server/evaluators-public-api.servertest.ts
@@ -0,0 +1,663 @@
+import { randomUUID } from "node:crypto";
+import { EvalTargetObject, EvalTemplateType } from "@langfuse/shared";
+import { prisma } from "@langfuse/shared/src/db";
+import { createOrgProjectAndApiKey } from "@langfuse/shared/src/server";
+import {
+ makeAPICall,
+ makeZodVerifiedAPICall,
+} from "@/src/__tests__/test-utils";
+import {
+ codeDefinition,
+ createCodeEvaluator,
+ createEvaluationRule,
+} from "@/src/__tests__/server/stable-evaluation-public-api-test-utils";
+import { EvaluatorService, RuleService } from "@/src/features/evals/server";
+import {
+ DeleteEvaluatorResponse,
+ Evaluator,
+ ListEvaluatorsResponse,
+ LlmAsJudgeEvaluator,
+ PublicApiError,
+} from "@/src/features/public-api";
+
+describe("stable evaluators public API", () => {
+ it("creates evaluators", async () => {
+ const { auth } = await createOrgProjectAndApiKey();
+
+ const first = await createCodeEvaluator("same name", auth);
+ const second = await createCodeEvaluator("same name", auth);
+ expect(first.body.id).not.toBe(second.body.id);
+ expect(first.body).toMatchObject({
+ name: "same name",
+ type: "code",
+ createdBy: null,
+ version: 1,
+ versionCreatedBy: null,
+ });
+ expect(first.body).not.toHaveProperty("latestVersion");
+
+ const llmEvaluator = await makeZodVerifiedAPICall(
+ Evaluator,
+ "POST",
+ "/api/public/v2/evaluators",
+ {
+ name: "friendly LLM evaluator",
+ type: "llm_as_judge",
+ prompt: "Input: {{input}}",
+ modelConfig: null,
+ variableMapping: null,
+ outputDefinition: {
+ dataType: "NUMERIC",
+ minValue: 0,
+ maxValue: 1,
+ },
+ },
+ auth,
+ 201,
+ );
+ const llmEvaluatorBody = LlmAsJudgeEvaluator.parse(llmEvaluator.body);
+ expect(llmEvaluatorBody).toMatchObject({
+ type: "llm_as_judge",
+ version: 1,
+ prompt: [{ role: "user", content: "Input: {{input}}" }],
+ variables: ["input"],
+ variableMapping: null,
+ modelConfig: null,
+ });
+ expect(llmEvaluatorBody.outputDefinition).toEqual({
+ dataType: "NUMERIC",
+ minValue: 0,
+ maxValue: 1,
+ });
+ await expect(
+ prisma.evaluatorVersion.findFirstOrThrow({
+ where: { evaluatorId: llmEvaluator.body.id },
+ orderBy: { version: "desc" },
+ select: { prompt: true },
+ }),
+ ).resolves.toEqual({ prompt: "Input: {{input}}" });
+
+ const categoricalEvaluator = await makeZodVerifiedAPICall(
+ Evaluator,
+ "POST",
+ "/api/public/v2/evaluators",
+ {
+ name: "categorical evaluator",
+ type: "llm_as_judge",
+ prompt: [{ role: "user", content: "Classify: {{input}}" }],
+ outputDefinition: {
+ dataType: "CATEGORICAL",
+ scoreReasoningInstructions: "Why the category was selected",
+ scoreValueInstructions: "The selected category",
+ categories: ["pass", "fail"],
+ shouldAllowMultipleMatches: false,
+ },
+ },
+ auth,
+ 201,
+ );
+ expect(
+ LlmAsJudgeEvaluator.parse(categoricalEvaluator.body).outputDefinition,
+ ).toEqual({
+ dataType: "CATEGORICAL",
+ scoreReasoningInstructions: "Why the category was selected",
+ scoreValueInstructions: "The selected category",
+ categories: ["pass", "fail"],
+ shouldAllowMultipleMatches: false,
+ });
+
+ const patchedEvaluator = await makeZodVerifiedAPICall(
+ Evaluator,
+ "PATCH",
+ `/api/public/v2/evaluators/${categoricalEvaluator.body.id}`,
+ {
+ type: "llm_as_judge",
+ prompt: "Classify updated: {{input}}",
+ outputDefinition: {
+ dataType: "BOOLEAN",
+ },
+ },
+ auth,
+ );
+ expect(LlmAsJudgeEvaluator.parse(patchedEvaluator.body).prompt).toEqual([
+ { role: "user", content: "Classify updated: {{input}}" },
+ ]);
+
+ const tooManyMessages = await makeAPICall(
+ "POST",
+ "/api/public/v2/evaluators",
+ {
+ name: "too many messages",
+ type: "llm_as_judge",
+ prompt: [
+ { role: "user", content: "First" },
+ { role: "user", content: "Second" },
+ ],
+ outputDefinition: { dataType: "BOOLEAN" },
+ },
+ auth,
+ );
+ expect(tooManyMessages.status).toBe(400);
+
+ for (const role of ["assistant", "system", "developer"]) {
+ const invalidRole = await makeAPICall(
+ "POST",
+ "/api/public/v2/evaluators",
+ {
+ name: "invalid role evaluator",
+ type: "llm_as_judge",
+ prompt: [{ role, content: "Judge {{input}}" }],
+ outputDefinition: { dataType: "BOOLEAN" },
+ },
+ auth,
+ );
+ expect(invalidRole.status).toBe(400);
+ expect(PublicApiError.parse(invalidRole.body)).toMatchObject({
+ code: "invalid_body",
+ });
+ }
+
+ const invalidModelParams = await makeAPICall(
+ "POST",
+ "/api/public/v2/evaluators",
+ {
+ name: "invalid model params",
+ type: "llm_as_judge",
+ prompt: "Judge {{input}}",
+ modelConfig: {
+ provider: "openai",
+ model: "gpt-4.1-mini",
+ modelParams: { temperature: 0 },
+ },
+ outputDefinition: { dataType: "BOOLEAN" },
+ },
+ auth,
+ );
+ expect(invalidModelParams.status).toBe(400);
+
+ const invalid = await makeAPICall(
+ "POST",
+ "/api/public/v2/evaluators",
+ { name: "missing type" },
+ auth,
+ );
+ expect(invalid.status).toBe(400);
+ expect(PublicApiError.parse(invalid.body)).toMatchObject({
+ code: "invalid_body",
+ details: { issues: expect.any(Array) },
+ });
+
+ const nestedDefinition = await makeAPICall(
+ "POST",
+ "/api/public/v2/evaluators",
+ {
+ name: "nested definition",
+ definition: codeDefinition("function evaluate() {}"),
+ },
+ auth,
+ );
+ expect(nestedDefinition.status).toBe(400);
+ });
+
+ it("gets an evaluator", async () => {
+ const { auth, projectId } = await createOrgProjectAndApiKey();
+ const user = await prisma.user.create({
+ data: {
+ id: randomUUID(),
+ name: "Evaluator Author",
+ email: `${randomUUID()}@example.com`,
+ },
+ });
+ const evaluator = await new EvaluatorService(
+ prisma,
+ async () => undefined,
+ ).create(
+ {
+ projectId,
+ name: "authored evaluator",
+ description: null,
+ definition: {
+ type: EvalTemplateType.LLM_AS_JUDGE,
+ prompt: "Judge {{input}}",
+ vars: ["input"],
+ provider: null,
+ model: null,
+ modelParams: null,
+ variableMapping: null,
+ outputDefinition: {
+ dataType: "NUMERIC",
+ reasoning: { description: "Explain the score" },
+ score: { description: "Return the score" },
+ },
+ },
+ },
+ user.id,
+ );
+ const rule = await new RuleService(prisma, async () => undefined).create(
+ {
+ projectId,
+ name: "associated rule",
+ targetObject: EvalTargetObject.EVENT,
+ enabled: false,
+ sampling: 1,
+ filter: [],
+ evaluatorAssignments: [
+ {
+ evaluatorId: evaluator.id,
+ variableMapping: [
+ { templateVariable: "input", selectedColumnId: "input" },
+ ],
+ },
+ ],
+ },
+ user.id,
+ );
+
+ const legacyRule = await prisma.evaluationRule.create({
+ data: {
+ projectId,
+ name: "legacy associated rule",
+ targetObject: EvalTargetObject.TRACE,
+ status: "INACTIVE",
+ filter: [],
+ sampling: 1,
+ delay: 0,
+ timeScope: ["NEW"],
+ },
+ });
+ await prisma.evaluationRuleEvaluatorAssignment.create({
+ data: {
+ projectId,
+ evaluationRuleId: legacyRule.id,
+ evaluatorId: evaluator.id,
+ variableMapping: [
+ {
+ templateVariable: "input",
+ langfuseObject: "trace",
+ objectName: null,
+ selectedColumnId: "input",
+ jsonSelector: null,
+ },
+ ],
+ },
+ });
+
+ const response = await makeZodVerifiedAPICall(
+ Evaluator,
+ "GET",
+ `/api/public/v2/evaluators/${evaluator.id}`,
+ undefined,
+ auth,
+ );
+ expect(response.body.createdBy).toEqual({
+ id: user.id,
+ name: user.name,
+ });
+ expect(response.body.versionCreatedBy).toEqual(response.body.createdBy);
+ const expectedAssignments = expect.arrayContaining([
+ {
+ evaluationRuleId: rule.id,
+ variableMappingOverride: [{ variable: "input", source: "input" }],
+ },
+ {
+ evaluationRuleId: legacyRule.id,
+ variableMappingOverride: [
+ {
+ mappingType: "legacy",
+ variable: "input",
+ langfuseObject: "trace",
+ objectName: null,
+ source: "input",
+ },
+ ],
+ },
+ ]);
+ expect(response.body.evaluationRuleAssignments).toEqual(
+ expectedAssignments,
+ );
+
+ const listResponse = await makeZodVerifiedAPICall(
+ ListEvaluatorsResponse,
+ "GET",
+ "/api/public/v2/evaluators",
+ undefined,
+ auth,
+ );
+ expect(
+ listResponse.body.data.find(({ id }) => id === evaluator.id)
+ ?.evaluationRuleAssignments,
+ ).toEqual(expectedAssignments);
+
+ const legacyOutputEvaluator = await prisma.evaluator.create({
+ data: {
+ projectId,
+ name: "legacy output evaluator",
+ description: null,
+ type: EvalTemplateType.LLM_AS_JUDGE,
+ versions: {
+ create: {
+ version: 1,
+ prompt: "Judge {{input}}",
+ vars: ["input"],
+ outputDefinition: { reasoning: "", score: "" },
+ },
+ },
+ },
+ });
+ const legacyResponse = await makeZodVerifiedAPICall(
+ Evaluator,
+ "GET",
+ `/api/public/v2/evaluators/${legacyOutputEvaluator.id}`,
+ undefined,
+ auth,
+ );
+ expect(legacyResponse.body).toMatchObject({
+ version: 1,
+ prompt: [{ role: "user", content: "Judge {{input}}" }],
+ outputDefinition: {
+ dataType: "NUMERIC",
+ },
+ });
+ });
+
+ it("returns 404 when getting an invalid evaluator ID", async () => {
+ const { auth } = await createOrgProjectAndApiKey();
+
+ const response = await makeAPICall(
+ "GET",
+ `/api/public/v2/evaluators/${randomUUID()}`,
+ undefined,
+ auth,
+ );
+ expect(response.status).toBe(404);
+ expect(PublicApiError.parse(response.body)).toMatchObject({
+ code: "resource_not_found",
+ message: "Evaluator not found",
+ });
+ });
+
+ it("lists evaluators", async () => {
+ const { auth } = await createOrgProjectAndApiKey();
+ const created = await createCodeEvaluator("listed evaluator", auth);
+
+ const response = await makeZodVerifiedAPICall(
+ ListEvaluatorsResponse,
+ "GET",
+ "/api/public/v2/evaluators",
+ undefined,
+ auth,
+ );
+ expect(response.body.data).toContainEqual(
+ expect.objectContaining({
+ id: created.body.id,
+ name: "listed evaluator",
+ }),
+ );
+
+ const invalidQuery = await makeAPICall(
+ "GET",
+ "/api/public/v2/evaluators?limit=101",
+ undefined,
+ auth,
+ );
+ expect(invalidQuery.status).toBe(400);
+ expect(PublicApiError.parse(invalidQuery.body)).toMatchObject({
+ code: "invalid_query",
+ details: { issues: expect.any(Array) },
+ });
+ });
+
+ it("lists evaluators with cursor pagination", async () => {
+ const { auth } = await createOrgProjectAndApiKey();
+ const created = await Promise.all([
+ createCodeEvaluator("cursor one", auth),
+ createCodeEvaluator("cursor two", auth),
+ createCodeEvaluator("cursor three", auth),
+ ]);
+
+ const firstPage = await makeZodVerifiedAPICall(
+ ListEvaluatorsResponse,
+ "GET",
+ "/api/public/v2/evaluators?limit=2",
+ undefined,
+ auth,
+ );
+ expect(firstPage.body.data).toHaveLength(2);
+ expect(firstPage.body.meta.cursor).toEqual(expect.any(String));
+
+ const secondPage = await makeZodVerifiedAPICall(
+ ListEvaluatorsResponse,
+ "GET",
+ `/api/public/v2/evaluators?limit=2&cursor=${encodeURIComponent(firstPage.body.meta.cursor!)}`,
+ undefined,
+ auth,
+ );
+ expect(secondPage.body.data).toHaveLength(1);
+ expect(secondPage.body.meta.cursor).toBeUndefined();
+
+ const listedIds = [...firstPage.body.data, ...secondPage.body.data].map(
+ ({ id }) => id,
+ );
+ expect(new Set(listedIds)).toEqual(
+ new Set(created.map(({ body }) => body.id)),
+ );
+ });
+
+ it("preserves model configuration when patching another field", async () => {
+ const { auth, projectId } = await createOrgProjectAndApiKey();
+ const evaluator = await new EvaluatorService(
+ prisma,
+ async () => undefined,
+ ).create(
+ {
+ projectId,
+ name: "configured evaluator",
+ description: null,
+ definition: {
+ type: EvalTemplateType.LLM_AS_JUDGE,
+ prompt: "Judge {{input}}",
+ vars: ["input"],
+ provider: "openai",
+ model: "gpt-4.1-mini",
+ modelParams: { temperature: 0.2 },
+ variableMapping: null,
+ outputDefinition: {
+ dataType: "BOOLEAN",
+ reasoning: { description: "Explain the score" },
+ score: { description: "Return the score" },
+ },
+ },
+ },
+ null,
+ );
+
+ await makeZodVerifiedAPICall(
+ Evaluator,
+ "PATCH",
+ `/api/public/v2/evaluators/${evaluator.id}`,
+ {
+ type: "llm_as_judge",
+ prompt: "Judge carefully: {{input}}",
+ outputDefinition: { dataType: "BOOLEAN" },
+ },
+ auth,
+ );
+
+ await expect(
+ prisma.evaluatorVersion.findFirstOrThrow({
+ where: { evaluatorId: evaluator.id },
+ orderBy: { version: "desc" },
+ select: {
+ provider: true,
+ model: true,
+ modelParams: true,
+ },
+ }),
+ ).resolves.toEqual({
+ provider: "openai",
+ model: "gpt-4.1-mini",
+ modelParams: { temperature: 0.2 },
+ });
+ });
+
+ it("patches an evaluator", async () => {
+ const { auth } = await createOrgProjectAndApiKey();
+ const evaluator = await createCodeEvaluator("concurrent evaluator", auth);
+ const rule = await createEvaluationRule({
+ name: "recovery assignment",
+ evaluatorId: evaluator.body.id,
+ auth,
+ });
+ await prisma.evaluator.update({
+ where: { id: evaluator.body.id },
+ data: {
+ blockedAt: new Date(),
+ blockReason: "EVAL_MODEL_CONFIG_INVALID",
+ blockMessage: "Invalid model configuration",
+ },
+ });
+
+ const recovered = await makeZodVerifiedAPICall(
+ Evaluator,
+ "PATCH",
+ `/api/public/v2/evaluators/${evaluator.body.id}`,
+ codeDefinition("function evaluate() { return { scores: [] }; }"),
+ auth,
+ );
+ expect(recovered.body).toMatchObject({
+ id: evaluator.body.id,
+ status: "active",
+ pausedAt: null,
+ pausedReason: null,
+ pausedMessage: null,
+ version: 1,
+ evaluationRuleAssignments: [{ evaluationRuleId: rule.body.id }],
+ });
+
+ const metadataUpdate = await makeZodVerifiedAPICall(
+ Evaluator,
+ "PATCH",
+ `/api/public/v2/evaluators/${evaluator.body.id}`,
+ { description: "metadata-only description" },
+ auth,
+ );
+ expect(metadataUpdate.body.version).toBe(1);
+
+ const partialDefinition = await makeAPICall(
+ "PATCH",
+ `/api/public/v2/evaluators/${evaluator.body.id}`,
+ { type: "code", sourceCode: "function evaluate() {}" },
+ auth,
+ );
+ expect(partialDefinition.status).toBe(400);
+ expect(PublicApiError.parse(partialDefinition.body).code).toBe(
+ "invalid_body",
+ );
+
+ const changedSource =
+ "function evaluate() { return { scores: [{ name: 'ok', value: true }] }; }";
+ await Promise.all([
+ makeZodVerifiedAPICall(
+ Evaluator,
+ "PATCH",
+ `/api/public/v2/evaluators/${evaluator.body.id}`,
+ { name: "concurrent renamed" },
+ auth,
+ ),
+ makeZodVerifiedAPICall(
+ Evaluator,
+ "PATCH",
+ `/api/public/v2/evaluators/${evaluator.body.id}`,
+ { description: "concurrent description" },
+ auth,
+ ),
+ makeZodVerifiedAPICall(
+ Evaluator,
+ "PATCH",
+ `/api/public/v2/evaluators/${evaluator.body.id}`,
+ codeDefinition(changedSource),
+ auth,
+ ),
+ ]);
+
+ const current = await makeZodVerifiedAPICall(
+ Evaluator,
+ "GET",
+ `/api/public/v2/evaluators/${evaluator.body.id}`,
+ undefined,
+ auth,
+ );
+ expect(current.body).toMatchObject({
+ name: "concurrent renamed",
+ description: "concurrent description",
+ version: 2,
+ sourceCode: changedSource,
+ });
+
+ const typeConflict = await makeAPICall(
+ "PATCH",
+ `/api/public/v2/evaluators/${evaluator.body.id}`,
+ {
+ type: "llm_as_judge",
+ prompt: "Input: {{input}}",
+ modelConfig: null,
+ variableMapping: null,
+ outputDefinition: {
+ dataType: "BOOLEAN",
+ },
+ },
+ auth,
+ );
+ expect(typeConflict.status).toBe(409);
+ expect(PublicApiError.parse(typeConflict.body)).toMatchObject({
+ code: "conflict",
+ message: "Evaluator type cannot be changed",
+ });
+ });
+
+ it("returns 404 when patching an invalid evaluator ID", async () => {
+ const { auth } = await createOrgProjectAndApiKey();
+
+ const response = await makeAPICall(
+ "PATCH",
+ `/api/public/v2/evaluators/${randomUUID()}`,
+ { name: "missing evaluator" },
+ auth,
+ );
+ expect(response.status).toBe(404);
+ });
+
+ it("deletes an evaluator", async () => {
+ const { auth } = await createOrgProjectAndApiKey();
+ const evaluator = await createCodeEvaluator("deleted evaluator", auth);
+
+ const response = await makeZodVerifiedAPICall(
+ DeleteEvaluatorResponse,
+ "DELETE",
+ `/api/public/v2/evaluators/${evaluator.body.id}`,
+ undefined,
+ auth,
+ );
+ expect(response.body.id).toBe(evaluator.body.id);
+
+ const get = await makeAPICall(
+ "GET",
+ `/api/public/v2/evaluators/${evaluator.body.id}`,
+ undefined,
+ auth,
+ );
+ expect(get.status).toBe(404);
+ });
+
+ it("returns 404 when deleting an invalid evaluator ID", async () => {
+ const { auth } = await createOrgProjectAndApiKey();
+
+ const response = await makeAPICall(
+ "DELETE",
+ `/api/public/v2/evaluators/${randomUUID()}`,
+ undefined,
+ auth,
+ );
+ expect(response.status).toBe(404);
+ });
+});
diff --git a/web/src/__tests__/server/stable-evaluation-public-api-test-utils.ts b/web/src/__tests__/server/stable-evaluation-public-api-test-utils.ts
new file mode 100644
index 0000000000..d6345f7d77
--- /dev/null
+++ b/web/src/__tests__/server/stable-evaluation-public-api-test-utils.ts
@@ -0,0 +1,43 @@
+import { makeZodVerifiedAPICall } from "@/src/__tests__/test-utils";
+import { EvaluationRule, Evaluator } from "@/src/features/public-api";
+
+export const codeDefinition = (sourceCode: string) => ({
+ type: "code" as const,
+ sourceCode,
+ sourceCodeLanguage: "TYPESCRIPT" as const,
+});
+
+export const createCodeEvaluator = (name: string, auth: string) =>
+ makeZodVerifiedAPICall(
+ Evaluator,
+ "POST",
+ "/api/public/v2/evaluators",
+ {
+ name,
+ description: `${name} description`,
+ ...codeDefinition("function evaluate() { return { scores: [] }; }"),
+ },
+ auth,
+ 201,
+ );
+
+export const createEvaluationRule = (params: {
+ name: string;
+ evaluatorId: string;
+ auth: string;
+}) =>
+ makeZodVerifiedAPICall(
+ EvaluationRule,
+ "POST",
+ "/api/public/v2/evaluation-rules",
+ {
+ name: params.name,
+ enabled: false,
+ filter: [],
+ evaluatorAssignments: [
+ { evaluatorId: params.evaluatorId, variableMapping: null },
+ ],
+ },
+ params.auth,
+ 201,
+ );
diff --git a/web/src/__tests__/server/unit/create-authed-project-api-route-auth-errors.servertest.ts b/web/src/__tests__/server/unit/create-authed-project-api-route-auth-errors.servertest.ts
index 901882a51d..ad7ee6b232 100644
--- a/web/src/__tests__/server/unit/create-authed-project-api-route-auth-errors.servertest.ts
+++ b/web/src/__tests__/server/unit/create-authed-project-api-route-auth-errors.servertest.ts
@@ -8,16 +8,16 @@ const {
mockRateLimitRequest,
mockTraceException,
mockLoggerDebug,
- mockCreateUnstablePublicApiAuthError,
- mockSendUnstablePublicApiErrorResponse,
+ mockCreateStructuredPublicApiAuthError,
+ mockSendStructuredPublicApiErrorResponse,
} = vi.hoisted(() => ({
mockVerifyAuthHeaderAndReturnScope: vi.fn(),
mockIsPrismaException: vi.fn(),
mockRateLimitRequest: vi.fn(),
mockTraceException: vi.fn(),
mockLoggerDebug: vi.fn(),
- mockCreateUnstablePublicApiAuthError: vi.fn((value) => value),
- mockSendUnstablePublicApiErrorResponse: vi.fn(),
+ mockCreateStructuredPublicApiAuthError: vi.fn((value) => value),
+ mockSendStructuredPublicApiErrorResponse: vi.fn(),
}));
vi.mock("@/src/features/public-api/server/apiAuth", () => ({
@@ -60,12 +60,13 @@ vi.mock("@/src/features/public-api/server/withMiddlewares", () => ({
}));
vi.mock(
- "@/src/features/public-api/server/unstable-public-api-error-contract",
+ "@/src/features/public-api/server/structuredPublicApiErrorContract",
() => ({
- unstablePublicEvalsErrorContract: "unstable-public-evals",
- createUnstablePublicApiAuthError: mockCreateUnstablePublicApiAuthError,
- createUnstablePublicApiRequestValidationError: vi.fn(),
- sendUnstablePublicApiErrorResponse: mockSendUnstablePublicApiErrorResponse,
+ structuredPublicApiErrorContract: "structured",
+ createStructuredPublicApiAuthError: mockCreateStructuredPublicApiAuthError,
+ createStructuredPublicApiRequestValidationError: vi.fn(),
+ sendStructuredPublicApiErrorResponse:
+ mockSendStructuredPublicApiErrorResponse,
}),
);
@@ -112,7 +113,7 @@ describe("createAuthedProjectAPIRoute auth error handling", () => {
});
async function callRoute(options?: {
- useUnstableErrorContract?: boolean;
+ useStructuredErrorContract?: boolean;
rateLimitUpgradePath?: {
legacyEndpoint: string;
replacementEndpoint: string;
@@ -124,8 +125,8 @@ describe("createAuthedProjectAPIRoute auth error handling", () => {
name: "Test Route",
querySchema: z.object({}),
responseSchema: z.object({ ok: z.literal(true) }),
- errorContract: options?.useUnstableErrorContract
- ? "unstable-public-evals"
+ errorContract: options?.useStructuredErrorContract
+ ? "structured"
: undefined,
rateLimitUpgradePath: options?.rateLimitUpgradePath,
fn: async () => ({ ok: true as const }),
@@ -183,19 +184,34 @@ describe("createAuthedProjectAPIRoute auth error handling", () => {
expect(mockTraceException).toHaveBeenCalledWith(prismaLikeError);
});
- it("returns unstable auth errors and traces prisma auth failures", async () => {
+ it("returns structured authentication errors", async () => {
+ mockVerifyAuthHeaderAndReturnScope.mockResolvedValueOnce({
+ validKey: false,
+ error: "Invalid credentials",
+ });
+
+ await callRoute({ useStructuredErrorContract: true });
+
+ expect(mockCreateStructuredPublicApiAuthError).toHaveBeenCalledWith({
+ statusCode: 401,
+ message: "Invalid credentials",
+ });
+ expect(mockSendStructuredPublicApiErrorResponse).toHaveBeenCalledTimes(1);
+ });
+
+ it("returns structured errors and traces prisma auth failures", async () => {
const prismaLikeError = new Error("Can't reach database server");
mockVerifyAuthHeaderAndReturnScope.mockRejectedValueOnce(prismaLikeError);
mockIsPrismaException.mockReturnValue(true);
- await callRoute({ useUnstableErrorContract: true });
+ await callRoute({ useStructuredErrorContract: true });
expect(mockTraceException).toHaveBeenCalledWith(prismaLikeError);
- expect(mockCreateUnstablePublicApiAuthError).toHaveBeenCalledWith({
+ expect(mockCreateStructuredPublicApiAuthError).toHaveBeenCalledWith({
statusCode: 503,
message: "Service Unavailable",
});
- expect(mockSendUnstablePublicApiErrorResponse).toHaveBeenCalledTimes(1);
+ expect(mockSendStructuredPublicApiErrorResponse).toHaveBeenCalledTimes(1);
});
it("keeps the shared rate limit response for routes without upgrade guidance", async () => {
diff --git a/web/src/__tests__/server/unit/dashboard-widget-min-version.servertest.ts b/web/src/__tests__/server/unit/dashboard-widget-min-version.servertest.ts
index 6a91a557aa..133b4a8633 100644
--- a/web/src/__tests__/server/unit/dashboard-widget-min-version.servertest.ts
+++ b/web/src/__tests__/server/unit/dashboard-widget-min-version.servertest.ts
@@ -2,7 +2,7 @@ import {
normalizePublicDashboardWidgetInput,
validatePublicDashboardWidgetInput,
} from "@/src/features/widgets/server/public-dashboard-widget-service";
-import { UnstablePublicApiError } from "@/src/features/public-api/server/unstable-public-api-error-contract";
+import { StructuredPublicApiError } from "@/src/features/public-api";
import { env as sharedEnv } from "@langfuse/shared/src/env";
const baseInput = {
@@ -82,8 +82,8 @@ describe("public dashboard widget version validation", () => {
validationError = error;
}
- expect(validationError).toBeInstanceOf(UnstablePublicApiError);
- const allowedValues = (validationError as UnstablePublicApiError).details
+ expect(validationError).toBeInstanceOf(StructuredPublicApiError);
+ const allowedValues = (validationError as StructuredPublicApiError).details
?.allowedValues;
expect(allowedValues?.length).toBeGreaterThan(0);
diff --git a/web/src/__tests__/server/unit/evaluator-input.servertest.ts b/web/src/__tests__/server/unit/evaluator-input.servertest.ts
new file mode 100644
index 0000000000..a03a62897c
--- /dev/null
+++ b/web/src/__tests__/server/unit/evaluator-input.servertest.ts
@@ -0,0 +1,95 @@
+import { EvalTemplateType } from "@langfuse/shared";
+import { describe, expect, it } from "vitest";
+import { z } from "zod";
+import {
+ McpEvaluatorInput,
+ McpEvaluatorInputBase,
+} from "@/src/features/mcp/features/evals/tools/evaluatorInput";
+
+const llmEvaluatorInput = {
+ name: "Answer quality",
+ type: EvalTemplateType.LLM_AS_JUDGE,
+ prompt: "Judge {{output}}",
+ outputDefinition: {
+ dataType: "NUMERIC",
+ reasoning: { description: "Explain the score" },
+ score: { description: "Return the score" },
+ },
+};
+
+const hasJsonSchemaComposition = (value: unknown): boolean => {
+ if (typeof value !== "object" || value === null) return false;
+ if (Array.isArray(value)) return value.some(hasJsonSchemaComposition);
+
+ const schema = value as Record<string, unknown>;
+ if (
+ Array.isArray(schema.anyOf) ||
+ Array.isArray(schema.oneOf) ||
+ Array.isArray(schema.allOf)
+ ) {
+ return true;
+ }
+
+ return Object.values(schema).some(hasJsonSchemaComposition);
+};
+
+describe("MCP evaluator input", () => {
+ it("exposes a plain JSON schema without unions or intersections", () => {
+ const schema = z.toJSONSchema(McpEvaluatorInputBase, {
+ target: "draft-7",
+ unrepresentable: "any",
+ });
+
+ expect(schema.properties?.modelConfig).toMatchObject({
+ type: "object",
+ required: ["provider", "model"],
+ });
+ expect(hasJsonSchemaComposition(schema)).toBe(false);
+ });
+
+ it("derives a plain variable mapping schema from the shared schema", () => {
+ expect(
+ McpEvaluatorInput.safeParse({
+ ...llmEvaluatorInput,
+ variableMapping: [
+ {
+ templateVariable: "output",
+ selectedColumnId: "output",
+ jsonSelector: "$.answer",
+ },
+ ],
+ }).success,
+ ).toBe(true);
+ });
+
+ it("validates custom model configuration through its schema", () => {
+ const incomplete = McpEvaluatorInput.safeParse({
+ ...llmEvaluatorInput,
+ modelConfig: { modelParams: { temperature: 0.2 } },
+ });
+
+ expect(incomplete.success).toBe(false);
+ if (!incomplete.success) {
+ expect(incomplete.error.issues.map(({ path }) => path)).toEqual(
+ expect.arrayContaining([
+ ["modelConfig", "provider"],
+ ["modelConfig", "model"],
+ ]),
+ );
+ }
+
+ expect(
+ McpEvaluatorInput.safeParse({
+ ...llmEvaluatorInput,
+ modelConfig: {
+ provider: "openai",
+ model: "gpt-4.1-mini",
+ modelParams: {
+ temperature: 0.2,
+ providerOptions: { openai: { reasoningEffort: "low" } },
+ },
+ },
+ }).success,
+ ).toBe(true);
+ });
+});
diff --git a/web/src/__tests__/server/unit/evaluator-v2-validation.servertest.ts b/web/src/__tests__/server/unit/evaluator-v2-validation.servertest.ts
index c935df44f2..d9fc6b4fae 100644
--- a/web/src/__tests__/server/unit/evaluator-v2-validation.servertest.ts
+++ b/web/src/__tests__/server/unit/evaluator-v2-validation.servertest.ts
@@ -41,13 +41,9 @@ describe("evaluator configuration validation", () => {
definition: {
type: EvalTemplateType.LLM_AS_JUDGE,
prompt: "Judge {{output}}",
- provider: null,
- model: null,
- modelParams: null,
- vars: ["output"],
+ modelConfig: null,
variableMapping: null,
outputDefinition: {
- version: 2,
dataType: "NUMERIC",
score: { description: "Quality" },
reasoning: { description: "Reasoning" },
@@ -111,7 +107,6 @@ describe("evaluator configuration validation", () => {
vars: ["output"],
variableMapping: null,
outputDefinition: {
- version: 2,
dataType: "NUMERIC",
score: { description: "Quality" },
reasoning: { description: "Reasoning" },
@@ -161,7 +156,6 @@ describe("evaluator configuration validation", () => {
vars: ["output"],
variableMapping: null,
outputDefinition: {
- version: 2,
dataType: "NUMERIC",
score: { description: "Quality" },
reasoning: { description: "Reasoning" },
@@ -191,7 +185,6 @@ describe("evaluator configuration validation", () => {
},
],
outputDefinition: {
- version: 2,
dataType: "NUMERIC",
score: { description: "Quality" },
reasoning: { description: "Reasoning" },
@@ -219,7 +212,6 @@ describe("evaluator configuration validation", () => {
{ templateVariable: "output", selectedColumnId: "output" },
],
outputDefinition: {
- version: 2,
dataType: "NUMERIC",
score: { description: "Quality" },
reasoning: { description: "Reasoning" },
@@ -249,7 +241,6 @@ describe("evaluator configuration validation", () => {
},
],
outputDefinition: {
- version: 2,
dataType: "NUMERIC",
score: { description: "Quality" },
reasoning: { description: "Reasoning" },
diff --git a/web/src/__tests__/server/unit/python-sdk-deprecations.servertest.ts b/web/src/__tests__/server/unit/python-sdk-deprecations.servertest.ts
index 6d92c99f44..6e5538249e 100644
--- a/web/src/__tests__/server/unit/python-sdk-deprecations.servertest.ts
+++ b/web/src/__tests__/server/unit/python-sdk-deprecations.servertest.ts
@@ -118,6 +118,15 @@ describe("Python SDK deprecations", () => {
"sessions:get",
"trace:get",
"trace:list",
+ "unstable/evaluation_rules:create",
+ "unstable/evaluation_rules:list",
+ "unstable/evaluation_rules:get",
+ "unstable/evaluation_rules:update",
+ "unstable/evaluation_rules:delete",
+ "unstable/evaluators:create",
+ "unstable/evaluators:list",
+ "unstable/evaluators:get",
+ "unstable/evaluators:delete",
]);
});
diff --git a/web/src/__tests__/server/unit/rate-limit-response.servertest.ts b/web/src/__tests__/server/unit/rate-limit-response.servertest.ts
index 785c12b850..5793c50a48 100644
--- a/web/src/__tests__/server/unit/rate-limit-response.servertest.ts
+++ b/web/src/__tests__/server/unit/rate-limit-response.servertest.ts
@@ -3,10 +3,10 @@ import { createMocks } from "node-mocks-http";
import type { RateLimitResult } from "@langfuse/shared";
const {
- mockCreateUnstablePublicApiRateLimitError,
- mockSendUnstablePublicApiErrorResponse,
+ mockCreateStructuredPublicApiRateLimitError,
+ mockSendStructuredPublicApiErrorResponse,
} = vi.hoisted(() => ({
- mockCreateUnstablePublicApiRateLimitError: vi.fn(
+ mockCreateStructuredPublicApiRateLimitError: vi.fn(
(
rateLimitRes: RateLimitResult,
options?: {
@@ -33,7 +33,7 @@ const {
};
},
),
- mockSendUnstablePublicApiErrorResponse: vi.fn((res, error) =>
+ mockSendStructuredPublicApiErrorResponse: vi.fn((res, error) =>
res.status(error.httpCode).json({
message: error.message,
code: error.code,
@@ -74,12 +74,13 @@ vi.mock("@langfuse/shared/src/server", () => ({
}));
vi.mock(
- "@/src/features/public-api/server/unstable-public-api-error-contract",
+ "@/src/features/public-api/server/structuredPublicApiErrorContract",
() => ({
- unstablePublicEvalsErrorContract: "unstable-public-evals",
- createUnstablePublicApiRateLimitError:
- mockCreateUnstablePublicApiRateLimitError,
- sendUnstablePublicApiErrorResponse: mockSendUnstablePublicApiErrorResponse,
+ structuredPublicApiErrorContract: "structured",
+ createStructuredPublicApiRateLimitError:
+ mockCreateStructuredPublicApiRateLimitError,
+ sendStructuredPublicApiErrorResponse:
+ mockSendStructuredPublicApiErrorResponse,
}),
);
@@ -141,11 +142,11 @@ describe("sendRateLimitResponse", () => {
expect(res.getHeader("Retry-After")).toBe(3);
expect(res.getHeader("X-RateLimit-Limit")).toBe(10);
expect(res.getHeader("X-RateLimit-Remaining")).toBe(-1);
- expect(mockCreateUnstablePublicApiRateLimitError).toHaveBeenCalledWith(
+ expect(mockCreateStructuredPublicApiRateLimitError).toHaveBeenCalledWith(
rateLimitResult,
{},
);
- expect(mockSendUnstablePublicApiErrorResponse).toHaveBeenCalledTimes(1);
+ expect(mockSendStructuredPublicApiErrorResponse).toHaveBeenCalledTimes(1);
});
it("returns upgrade guidance for stable responses with an upgrade path", () => {
@@ -168,31 +169,31 @@ describe("sendRateLimitResponse", () => {
resetAt: expect.any(String),
},
});
- expect(mockCreateUnstablePublicApiRateLimitError).toHaveBeenCalledWith(
+ expect(mockCreateStructuredPublicApiRateLimitError).toHaveBeenCalledWith(
rateLimitResult,
{
upgradePath,
},
);
- expect(mockSendUnstablePublicApiErrorResponse).toHaveBeenCalledTimes(1);
+ expect(mockSendStructuredPublicApiErrorResponse).toHaveBeenCalledTimes(1);
});
it("passes the error contract through when an upgrade path is present", () => {
const res = createResponse();
sendRateLimitResponse(res, rateLimitResult, {
- errorContract: "unstable-public-evals",
+ errorContract: "structured",
upgradePath,
});
- expect(mockCreateUnstablePublicApiRateLimitError).toHaveBeenCalledWith(
+ expect(mockCreateStructuredPublicApiRateLimitError).toHaveBeenCalledWith(
rateLimitResult,
{
- errorContract: "unstable-public-evals",
+ errorContract: "structured",
upgradePath,
},
);
- expect(mockSendUnstablePublicApiErrorResponse).toHaveBeenCalledTimes(1);
+ expect(mockSendStructuredPublicApiErrorResponse).toHaveBeenCalledTimes(1);
expect(res.statusCode).toBe(429);
expect(res._getJSONData()).toEqual({
message:
diff --git a/web/src/__tests__/server/unit/unstable-public-api-error-contract.servertest.ts b/web/src/__tests__/server/unit/structured-public-api-error-contract.servertest.ts
similarity index 54%
rename from web/src/__tests__/server/unit/unstable-public-api-error-contract.servertest.ts
rename to web/src/__tests__/server/unit/structured-public-api-error-contract.servertest.ts
index 0f9c3a8ef7..d4223bf12b 100644
--- a/web/src/__tests__/server/unit/unstable-public-api-error-contract.servertest.ts
+++ b/web/src/__tests__/server/unit/structured-public-api-error-contract.servertest.ts
@@ -1,7 +1,38 @@
import type { RateLimitResult } from "@langfuse/shared";
-import { createUnstablePublicApiRateLimitError } from "@/src/features/public-api/server/unstable-public-api-error-contract";
+import { ClickHouseResourceError } from "@langfuse/shared/src/server";
+import { EvaluatorVersionConflictError } from "@/src/features/evals/v2/server/evaluators/evaluatorErrors";
+import {
+ createStructuredPublicApiRateLimitError,
+ structuredPublicApiErrorContract,
+ toStructuredPublicApiError,
+} from "@/src/features/public-api/server";
+
+describe("structured public api error contract", () => {
+ it("uses one contract marker", () => {
+ expect(structuredPublicApiErrorContract).toBe("structured");
+ });
+
+ it("maps evaluator version conflicts to the conflict code", () => {
+ const error = toStructuredPublicApiError(
+ new EvaluatorVersionConflictError(),
+ );
+
+ expect(error.name).toBe("StructuredPublicApiError");
+ expect(error.httpCode).toBe(409);
+ expect(error.code).toBe("conflict");
+ });
+
+ it("only emits shared codes for mapped errors", () => {
+ const resourceError = new ClickHouseResourceError(
+ "TIMEOUT",
+ new Error("timed out"),
+ );
+
+ expect(toStructuredPublicApiError(resourceError).code).toBe(
+ "invalid_request",
+ );
+ });
-describe("unstable public api error contract", () => {
it("supports upgrade path rate limit messages and clamps remaining points", () => {
const upgradePath = {
legacyEndpoint: "GET /api/public/traces",
@@ -30,7 +61,7 @@ describe("unstable public api error contract", () => {
isFirstInDuration: false,
} satisfies RateLimitResult;
- const error = createUnstablePublicApiRateLimitError(rateLimitResult, {
+ const error = createStructuredPublicApiRateLimitError(rateLimitResult, {
upgradePath,
});
diff --git a/web/src/__tests__/server/unit/typescript-sdk-deprecations.servertest.ts b/web/src/__tests__/server/unit/typescript-sdk-deprecations.servertest.ts
index 9ee7ac7ca1..43c9afd798 100644
--- a/web/src/__tests__/server/unit/typescript-sdk-deprecations.servertest.ts
+++ b/web/src/__tests__/server/unit/typescript-sdk-deprecations.servertest.ts
@@ -146,6 +146,15 @@ describe("TypeScript SDK deprecations", () => {
"api/resources/sessions/client/Client.ts:get",
"api/resources/trace/client/Client.ts:get",
"api/resources/trace/client/Client.ts:list",
+ "api/resources/unstable/resources/evaluationRules/client/Client.ts:create",
+ "api/resources/unstable/resources/evaluationRules/client/Client.ts:list",
+ "api/resources/unstable/resources/evaluationRules/client/Client.ts:get",
+ "api/resources/unstable/resources/evaluationRules/client/Client.ts:update",
+ "api/resources/unstable/resources/evaluationRules/client/Client.ts:delete",
+ "api/resources/unstable/resources/evaluators/client/Client.ts:create",
+ "api/resources/unstable/resources/evaluators/client/Client.ts:list",
+ "api/resources/unstable/resources/evaluators/client/Client.ts:get",
+ "api/resources/unstable/resources/evaluators/client/Client.ts:delete",
]);
expect(
deprecations
diff --git a/web/src/__tests__/server/unit/unstable-evals-contract.servertest.ts b/web/src/__tests__/server/unit/unstable-evals-contract.servertest.ts
index 408d8171da..bff8db6736 100644
--- a/web/src/__tests__/server/unit/unstable-evals-contract.servertest.ts
+++ b/web/src/__tests__/server/unit/unstable-evals-contract.servertest.ts
@@ -11,7 +11,7 @@ import {
toApiV2EvaluationRule,
toEvaluationRuleInput,
} from "@/src/features/evals/server/unstable-public-api/adapters";
-import { UnstablePublicApiError } from "@/src/features/public-api/server/unstable-public-api-error-contract";
+import { StructuredPublicApiError } from "@/src/features/public-api";
import type {
StoredPublicEvaluatorTemplate,
StoredPublicV2EvaluationRule,
@@ -37,21 +37,21 @@ const numericOutputDefinition = createNumericEvalOutputDefinition({
maxValue: 1,
});
-const expectUnstablePublicApiError = (
+const expectStructuredPublicApiError = (
fn: () => unknown,
params: {
- code: UnstablePublicApiError["code"];
+ code: StructuredPublicApiError["code"];
message?: string;
details?: Record<string, unknown>;
},
) => {
try {
fn();
- throw new Error("Expected function to throw UnstablePublicApiError");
+ throw new Error("Expected function to throw StructuredPublicApiError");
} catch (error) {
- expect(error).toBeInstanceOf(UnstablePublicApiError);
+ expect(error).toBeInstanceOf(StructuredPublicApiError);
- const unstableError = error as UnstablePublicApiError;
+ const unstableError = error as StructuredPublicApiError;
expect(unstableError.code).toBe(params.code);
if (params.message) {
@@ -659,7 +659,7 @@ describe("unstable public eval adapters", () => {
});
it("rejects invalid static filter option values", () => {
- expectUnstablePublicApiError(
+ expectStructuredPublicApiError(
() =>
toEvaluationRuleInput({
input: {
@@ -692,7 +692,7 @@ describe("unstable public eval adapters", () => {
});
it("rejects malformed jsonPath selectors", () => {
- expectUnstablePublicApiError(
+ expectStructuredPublicApiError(
() =>
toEvaluationRuleInput({
input: {
@@ -724,7 +724,7 @@ describe("unstable public eval adapters", () => {
});
it("rejects mapping sources that are incompatible with the selected target", () => {
- expectUnstablePublicApiError(
+ expectStructuredPublicApiError(
() =>
toEvaluationRuleInput({
input: {
@@ -778,7 +778,7 @@ describe("unstable public eval adapters", () => {
});
it("rejects missing evaluator variable mappings", () => {
- expectUnstablePublicApiError(
+ expectStructuredPublicApiError(
() =>
toEvaluationRuleInput({
input: {
diff --git a/web/src/__tests__/server/unstable-dashboard-widgets-api.servertest.ts b/web/src/__tests__/server/unstable-dashboard-widgets-api.servertest.ts
index 58e748898a..fa3f361771 100644
--- a/web/src/__tests__/server/unstable-dashboard-widgets-api.servertest.ts
+++ b/web/src/__tests__/server/unstable-dashboard-widgets-api.servertest.ts
@@ -8,7 +8,7 @@ import {
PatchUnstableDashboardWidgetResponse,
PostUnstableDashboardWidgetResponse,
} from "@/src/features/public-api/types/unstable-dashboard-widgets";
-import { UnstablePublicApiErrorResponse } from "@/src/features/public-api/types/unstable-public-evals-contract";
+import { StructuredPublicApiErrorResponse } from "@/src/features/public-api/types/unstable-public-evals-contract";
import { DashboardWidgetViews, prisma } from "@langfuse/shared/src/db";
import {
createOrgProjectAndApiKey,
@@ -41,11 +41,11 @@ const expectUnstableError = (
response: Awaited<ReturnType<typeof makeAPICall>>,
params: {
status: number;
- code: z.infer<typeof UnstablePublicApiErrorResponse>["code"];
+ code: z.infer<typeof StructuredPublicApiErrorResponse>["code"];
},
) => {
expect(response.status).toBe(params.status);
- const body = UnstablePublicApiErrorResponse.parse(response.body);
+ const body = StructuredPublicApiErrorResponse.parse(response.body);
expect(body.code).toBe(params.code);
return body;
};
diff --git a/web/src/__tests__/server/unstable-dashboards-api.servertest.ts b/web/src/__tests__/server/unstable-dashboards-api.servertest.ts
index 1dedeb6ad9..e9459c8694 100644
--- a/web/src/__tests__/server/unstable-dashboards-api.servertest.ts
+++ b/web/src/__tests__/server/unstable-dashboards-api.servertest.ts
@@ -19,7 +19,7 @@ import {
PostDashboardPlacementResponse,
PostUnstableDashboardResponse,
} from "@/src/features/public-api/types/unstable-dashboards";
-import { UnstablePublicApiErrorResponse } from "@/src/features/public-api/types/unstable-public-evals-contract";
+import { StructuredPublicApiErrorResponse } from "@/src/features/public-api/types/unstable-public-evals-contract";
import { createOrgProjectAndApiKey } from "@langfuse/shared/src/server";
import { prisma } from "@langfuse/shared/src/db";
import { nanoid } from "nanoid";
@@ -178,7 +178,7 @@ describe("unstable dashboard API", () => {
);
expect(duplicatePlacement.status).toBe(409);
expect(
- UnstablePublicApiErrorResponse.parse(duplicatePlacement.body).code,
+ StructuredPublicApiErrorResponse.parse(duplicatePlacement.body).code,
).toBe("conflict");
const blockedDelete = await makeAPICall(
@@ -188,9 +188,9 @@ describe("unstable dashboard API", () => {
auth,
);
expect(blockedDelete.status).toBe(409);
- expect(UnstablePublicApiErrorResponse.parse(blockedDelete.body).code).toBe(
- "conflict",
- );
+ expect(
+ StructuredPublicApiErrorResponse.parse(blockedDelete.body).code,
+ ).toBe("conflict");
const moved = await makeZodVerifiedAPICall(
PatchDashboardPlacementResponse,
diff --git a/web/src/__tests__/server/unstable-evals-api.servertest.ts b/web/src/__tests__/server/unstable-evals-api.servertest.ts
index 838afcebde..1b1c2d720a 100644
--- a/web/src/__tests__/server/unstable-evals-api.servertest.ts
+++ b/web/src/__tests__/server/unstable-evals-api.servertest.ts
@@ -26,7 +26,7 @@ import {
createOrgProjectAndApiKey,
} from "@langfuse/shared/src/server";
import { ApiKeyScope } from "@prisma/client";
-import { UnstablePublicApiErrorResponse } from "@/src/features/public-api/types/unstable-public-evals-contract";
+import { StructuredPublicApiErrorResponse } from "@/src/features/public-api/types/unstable-public-evals-contract";
import type { z } from "zod";
import { encrypt } from "@langfuse/shared/encryption";
@@ -71,11 +71,11 @@ const expectUnstableError = (
response: Awaited<ReturnType<typeof makeAPICall>>,
params: {
status: number;
- code: z.infer<typeof UnstablePublicApiErrorResponse>["code"];
+ code: z.infer<typeof StructuredPublicApiErrorResponse>["code"];
},
) => {
expect(response.status).toBe(params.status);
- const body = UnstablePublicApiErrorResponse.parse(response.body);
+ const body = StructuredPublicApiErrorResponse.parse(response.body);
expect(body.code).toBe(params.code);
return body;
};
diff --git a/web/src/features/audit-logs/server/index.ts b/web/src/features/audit-logs/server/index.ts
new file mode 100644
index 0000000000..4bd54c47e6
--- /dev/null
+++ b/web/src/features/audit-logs/server/index.ts
@@ -0,0 +1 @@
+export { auditLog } from "@/src/features/audit-logs/auditLog";
diff --git a/web/src/features/evals/server/index.ts b/web/src/features/evals/server/index.ts
new file mode 100644
index 0000000000..e8fdb7a0a3
--- /dev/null
+++ b/web/src/features/evals/server/index.ts
@@ -0,0 +1,8 @@
+export {
+ EVAL_TEMPLATE_AUDIT_LOG_RESOURCE_TYPE,
+ JOB_CONFIGURATION_AUDIT_LOG_RESOURCE_TYPE,
+} from "./audit-log-resource-types";
+export { EvaluatorService } from "@/src/features/evals/v2/server/evaluators/evaluatorService";
+export { EvaluatorDefinitionInputSchema } from "@/src/features/evals/v2/server/evaluators/evaluatorTypes";
+export { RuleService } from "@/src/features/evals/v2/server/rules/ruleService";
+export { isLegacyEvalTarget } from "@/src/features/evals/utils/typeHelpers";
diff --git a/web/src/features/evals/server/unstable-public-api/adapters.ts b/web/src/features/evals/server/unstable-public-api/adapters.ts
index 82a4f453d9..1cbb70820a 100644
--- a/web/src/features/evals/server/unstable-public-api/adapters.ts
+++ b/web/src/features/evals/server/unstable-public-api/adapters.ts
@@ -24,17 +24,17 @@ import { EvalTemplateType } from "@langfuse/shared/src/db";
import { logger } from "@langfuse/shared/src/server";
import { z } from "zod";
import {
- LegacyEvaluationRuleMapping,
+ LegacyPromptVariableMapping,
PUBLIC_EVALUATOR_TYPE_CODE,
PUBLIC_EVALUATOR_TYPE_LLM_AS_JUDGE,
PublicEvaluationRuleFilter,
type PublicEvaluationRuleFilterType,
- type PublicEvaluationRuleMappingType,
- type PublicEvaluationRuleReadMappingType,
+ type PromptVariableMappingInputType,
+ type PromptVariableMappingReadType,
type PublicEvaluationRuleReadTargetType,
type PublicEvaluationRuleStatusType,
type PublicEvaluationRuleTargetType,
- type LegacyEvaluationRuleMappingType,
+ type LegacyPromptVariableMappingType,
type PublicEvaluatorModelConfigType,
type PublicEvaluatorOutputDefinitionType,
type PublicEvaluatorTypeType,
@@ -81,7 +81,7 @@ const INTERNAL_TARGET_OBJECT_TO_PUBLIC_TARGET: Record<
};
const PUBLIC_MAPPING_SOURCE_TO_INTERNAL_COLUMN: Record<
- PublicEvaluationRuleMappingType["source"],
+ PromptVariableMappingInputType["source"],
ObservationVariableMapping["selectedColumnId"]
> = {
input: "input",
@@ -94,7 +94,7 @@ const PUBLIC_MAPPING_SOURCE_TO_INTERNAL_COLUMN: Record<
const INTERNAL_MAPPING_COLUMN_TO_PUBLIC_SOURCE: Record<
string,
- PublicEvaluationRuleMappingType["source"]
+ PromptVariableMappingInputType["source"]
> = {
input: "input",
output: "output",
@@ -121,7 +121,7 @@ export function deriveEvaluatorVariables(
}
export function toStoredVariableMappings(params: {
- mappings: PublicEvaluationRuleMappingType[];
+ mappings: PromptVariableMappingInputType[];
variables: string[];
target: PublicEvaluationRuleTargetType;
}) {
@@ -140,7 +140,7 @@ export function toStoredVariableMappings(params: {
* elsewhere; prefer `toStoredVariableMappings` when they are.
*/
export function toStoredMappingList(
- mappings: PublicEvaluationRuleMappingType[],
+ mappings: PromptVariableMappingInputType[],
) {
return observationVariableMappingList.parse(
mappings.map((mapping) => ({
@@ -297,7 +297,7 @@ function toApiFilter(filter: z.infer<typeof singleFilter>) {
export function toApiReadMappings(
mappings: unknown,
-): PublicEvaluationRuleReadMappingType[] {
+): PromptVariableMappingReadType[] {
const parsed = observationVariableMappingList.safeParse(mappings);
if (!parsed.success) {
@@ -325,7 +325,7 @@ export function toApiReadMappings(
export function toApiMappings(
mappings: unknown,
-): PublicEvaluationRuleMappingType[] {
+): PromptVariableMappingInputType[] {
return toApiReadMappings(mappings).map((mapping) => {
if (mapping.source === null) {
throw new InternalServerError("Evaluation rule mapping is corrupted");
@@ -337,7 +337,7 @@ export function toApiMappings(
function toApiLegacyMappings(
mappings: unknown,
-): LegacyEvaluationRuleMappingType[] {
+): LegacyPromptVariableMappingType[] {
const parsed = variableMappingList.safeParse(mappings);
if (!parsed.success) {
@@ -355,7 +355,7 @@ function toApiLegacyMappings(
...(mapping.jsonSelector ? { jsonPath: mapping.jsonSelector } : {}),
}));
const parsedPublicMappings = z
- .array(LegacyEvaluationRuleMapping)
+ .array(LegacyPromptVariableMapping)
.safeParse(publicMappings);
if (!parsedPublicMappings.success) {
logger.error("Failed to parse unstable public legacy rule mappings", {
@@ -369,7 +369,7 @@ function toApiLegacyMappings(
function toDefaultLegacyMappings(
mappings: unknown,
target: "trace" | "dataset",
-): LegacyEvaluationRuleMappingType[] {
+): LegacyPromptVariableMappingType[] {
return toApiMappings(mappings).map((mapping) => ({
...mapping,
langfuseObject: target === "trace" ? "trace" : "dataset_item",
@@ -624,7 +624,7 @@ export function toEvaluationRuleInput(params: {
enabled: boolean;
sampling: number;
filter: PublicEvaluationRuleFilterType[];
- mapping?: PublicEvaluationRuleMappingType[];
+ mapping?: PromptVariableMappingInputType[];
};
evaluatorVariables: string[];
evaluatorType: PublicEvaluatorTypeType;
@@ -667,7 +667,7 @@ export function toEvaluationRuleFields(input: {
/** Stored mapping for one assignment. Code evaluators inherit their fixed evaluator mapping. */
export function toStoredAssignmentMapping(params: {
target: PublicEvaluationRuleTargetType;
- mapping?: PublicEvaluationRuleMappingType[];
+ mapping?: PromptVariableMappingInputType[];
evaluatorVariables: string[];
evaluatorType: PublicEvaluatorTypeType;
}) {
diff --git a/web/src/features/evals/server/unstable-public-api/evaluation-rule-service.ts b/web/src/features/evals/server/unstable-public-api/evaluation-rule-service.ts
index 40cef77bc6..c5161c67ba 100644
--- a/web/src/features/evals/server/unstable-public-api/evaluation-rule-service.ts
+++ b/web/src/features/evals/server/unstable-public-api/evaluation-rule-service.ts
@@ -23,8 +23,8 @@ import type {
PostUnstableEvaluationRuleBodyType,
} from "@/src/features/public-api/types/unstable-evaluation-rules";
import {
- type PublicEvaluationRuleMappingType,
- type PublicEvaluationRuleReadMappingType,
+ type PromptVariableMappingInputType,
+ type PromptVariableMappingReadType,
type PublicEvaluationRuleTargetType,
PUBLIC_EVALUATOR_TYPE_LLM_AS_JUDGE,
} from "@/src/features/public-api/types/unstable-public-evals-contract";
@@ -58,7 +58,7 @@ import {
assertEvaluationRuleFilterValuesExistForProject,
assertEvaluatorDefinitionCanRunForPublicApi,
} from "./validation";
-import { createUnstablePublicApiError } from "@/src/features/public-api/server/unstable-public-api-error-contract";
+import { createStructuredPublicApiError } from "@/src/features/public-api";
import { assertUnreachable } from "@/src/utils/types";
const ruleService = new RuleService(prisma, async () => undefined);
@@ -79,7 +79,7 @@ function toEvaluatorDefinition(
): StoredPublicEvaluatorTemplate {
const version = evaluator.versions[0];
if (!version) {
- throw createUnstablePublicApiError({
+ throw createStructuredPublicApiError({
httpCode: 500,
code: "internal_error",
message: "Evaluator version is missing",
@@ -147,7 +147,7 @@ async function assertEvaluationRuleCanRunForPublicApi(params: {
}
if (!isCodeEvalEnabled()) {
- throw createUnstablePublicApiError({
+ throw createStructuredPublicApiError({
httpCode: 403,
code: "access_denied",
message: "Code evals are not enabled",
@@ -157,7 +157,7 @@ async function assertEvaluationRuleCanRunForPublicApi(params: {
if (
!isCodeEvalSourceCodeLanguageSupported(params.template.sourceCodeLanguage)
) {
- throw createUnstablePublicApiError({
+ throw createStructuredPublicApiError({
httpCode: 400,
code: "invalid_request",
message:
@@ -183,21 +183,21 @@ async function assertEvaluationRuleCanRunForPublicApi(params: {
switch (error.code) {
case "invalid_target":
case "invalid_request":
- throw createUnstablePublicApiError({
+ throw createStructuredPublicApiError({
httpCode: 400,
code: "invalid_request",
message: error.message,
details,
});
case "resource_not_found":
- throw createUnstablePublicApiError({
+ throw createStructuredPublicApiError({
httpCode: 404,
code: "resource_not_found",
message: error.message,
details,
});
case "preflight_failed":
- throw createUnstablePublicApiError({
+ throw createStructuredPublicApiError({
httpCode: 422,
code: "evaluator_preflight_failed",
message: error.message,
@@ -227,7 +227,7 @@ async function assertActivePublicApiEvaluationRuleLimitNotExceeded(
function throwPublicRuleServiceError(error: unknown): never {
if (error instanceof ActiveEvaluationRuleLimitError) {
- throw createUnstablePublicApiError({
+ throw createStructuredPublicApiError({
httpCode: 409,
code: "conflict",
message: error.message,
@@ -321,7 +321,7 @@ export async function createPublicEvaluationRule(params: {
select: { id: true },
});
if (existing) {
- throw createUnstablePublicApiError({
+ throw createStructuredPublicApiError({
httpCode: 409,
code: "name_conflict",
message: `An evaluation rule named "${params.input.name}" already exists in this project. Use PATCH /api/public/unstable/evaluation-rules/${existing.id} to update it instead of creating a duplicate.`,
@@ -351,7 +351,7 @@ export async function createPublicEvaluationRule(params: {
),
);
if (new Set(evaluators.map(({ id }) => id)).size !== evaluators.length) {
- throw createUnstablePublicApiError({
+ throw createStructuredPublicApiError({
httpCode: 409,
code: "conflict",
message: "An evaluator can only be attached once to an evaluation rule.",
@@ -424,7 +424,7 @@ async function createRuleWithRuleService(params: {
data: ReturnType<typeof toEvaluationRuleInput>;
assignments: Array<{
evaluator: PublicV2Evaluator;
- assignment: { mapping?: PublicEvaluationRuleMappingType[] };
+ assignment: { mapping?: PromptVariableMappingInputType[] };
data: ReturnType<typeof toEvaluationRuleInput>;
}>;
}) {
@@ -466,9 +466,9 @@ type PreparedAssignment = {
};
function toWritableMappingsIfComplete(
- mappings: PublicEvaluationRuleReadMappingType[],
-): PublicEvaluationRuleMappingType[] | undefined {
- const completeMappings: PublicEvaluationRuleMappingType[] = [];
+ mappings: PromptVariableMappingReadType[],
+): PromptVariableMappingInputType[] | undefined {
+ const completeMappings: PromptVariableMappingInputType[] = [];
for (const mapping of mappings) {
if (mapping.source === null) return undefined;
completeMappings.push({ ...mapping, source: mapping.source });
@@ -484,7 +484,7 @@ async function prepareAssignment(params: {
projectId: string;
target: PublicEvaluationRuleTargetType;
evaluator: EvaluationRuleEvaluatorFamilyReference;
- mapping?: PublicEvaluationRuleMappingType[];
+ mapping?: PromptVariableMappingInputType[];
evaluatorId?: string;
}): Promise<PreparedAssignment> {
const evaluator = await resolveProjectEvaluator({
@@ -523,7 +523,7 @@ async function prepareAssignment(params: {
function assertUniqueEvaluators(assignments: Array<{ evaluatorId: string }>) {
const ids = assignments.map(({ evaluatorId }) => evaluatorId);
if (new Set(ids).size !== ids.length) {
- throw createUnstablePublicApiError({
+ throw createStructuredPublicApiError({
httpCode: 409,
code: "conflict",
message: "An evaluator can only be attached once to an evaluation rule.",
@@ -604,7 +604,7 @@ export async function updatePublicEvaluationRule(params: {
params.input.mapping !== undefined &&
firstAssignment?.evaluator.type === EvalTemplateType.CODE
) {
- throw createUnstablePublicApiError({
+ throw createStructuredPublicApiError({
httpCode: 400,
code: "invalid_body",
message:
@@ -613,7 +613,7 @@ export async function updatePublicEvaluationRule(params: {
});
}
if (patchesFirstAssignment && !firstAssignment && !params.input.evaluator) {
- throw createUnstablePublicApiError({
+ throw createStructuredPublicApiError({
httpCode: 400,
code: "invalid_body",
message:
diff --git a/web/src/features/evals/server/unstable-public-api/evaluator-service.ts b/web/src/features/evals/server/unstable-public-api/evaluator-service.ts
index c15794d313..b41db5975f 100644
--- a/web/src/features/evals/server/unstable-public-api/evaluator-service.ts
+++ b/web/src/features/evals/server/unstable-public-api/evaluator-service.ts
@@ -21,7 +21,7 @@ import {
EvaluatorVersionConflictError,
} from "@/src/features/evals/v2/server/evaluators/evaluatorErrors";
import { RuleService } from "@/src/features/evals/v2/server/rules/ruleService";
-import { createUnstablePublicApiError } from "@/src/features/public-api/server/unstable-public-api-error-contract";
+import { createStructuredPublicApiError } from "@/src/features/public-api";
import type { PostUnstableEvaluatorBodyParsedType } from "@/src/features/public-api/types/unstable-evaluators";
import { PUBLIC_EVALUATOR_TYPE_CODE } from "@/src/features/public-api/types/unstable-public-evals-contract";
import {
@@ -74,7 +74,7 @@ function assertCodeEvaluatorDefinitionCanRunForPublicApi(
>,
) {
if (!isCodeEvalEnabled()) {
- throw createUnstablePublicApiError({
+ throw createStructuredPublicApiError({
httpCode: 403,
code: "access_denied",
message: "Code evals are not enabled",
@@ -82,7 +82,7 @@ function assertCodeEvaluatorDefinitionCanRunForPublicApi(
}
if (!isCodeEvalSourceCodeLanguageSupported(input.sourceCodeLanguage)) {
- throw createUnstablePublicApiError({
+ throw createStructuredPublicApiError({
httpCode: 400,
code: "invalid_request",
message:
@@ -125,14 +125,16 @@ function toDefinition(
};
}
-type StableEvaluator = Awaited<ReturnType<EvaluatorService["get"]>>;
+type StableEvaluator =
+ | Awaited<ReturnType<EvaluatorService["get"]>>
+ | Awaited<ReturnType<EvaluatorService["list"]>>["evaluators"][number];
function toLatestTemplate(
evaluator: StableEvaluator,
): StoredPublicEvaluatorTemplate {
const version = evaluator.versions[0];
if (!version) {
- throw createUnstablePublicApiError({
+ throw createStructuredPublicApiError({
httpCode: 500,
code: "internal_error",
message: "Evaluator version is missing",
@@ -240,7 +242,7 @@ export async function createPublicEvaluator(params: {
return toPublicEvaluator(evaluator, ruleCounts[evaluator.id] ?? 0);
} catch (error) {
if (error instanceof EvaluatorConfigurationError) {
- throw createUnstablePublicApiError({
+ throw createStructuredPublicApiError({
httpCode: 422,
code: "evaluator_preflight_failed",
message: error.message,
@@ -258,14 +260,14 @@ export async function createPublicEvaluator(params: {
});
}
if (error instanceof EvaluatorVersionConflictError) {
- throw createUnstablePublicApiError({
+ throw createStructuredPublicApiError({
httpCode: 409,
code: "conflict",
message: error.message,
});
}
if (error instanceof LangfuseConflictError) {
- throw createUnstablePublicApiError({
+ throw createStructuredPublicApiError({
httpCode: 409,
code: "name_conflict",
message: error.message,
diff --git a/web/src/features/evals/server/unstable-public-api/types.ts b/web/src/features/evals/server/unstable-public-api/types.ts
index fcf9af67cf..0f6c779521 100644
--- a/web/src/features/evals/server/unstable-public-api/types.ts
+++ b/web/src/features/evals/server/unstable-public-api/types.ts
@@ -10,10 +10,10 @@ import type {
PublicEvaluationRuleEvaluatorReferenceType,
PublicEvaluationRuleEvaluatorType,
PublicEvaluationRuleFilterType,
- PublicEvaluationRuleReadMappingType,
+ PromptVariableMappingReadType,
PublicEvaluationRuleStatusType,
PublicEvaluationRuleTargetType,
- LegacyEvaluationRuleMappingType,
+ LegacyPromptVariableMappingType,
PublicEvaluationRuleLegacyTargetType,
PublicEvaluatorModelConfigType,
PublicEvaluatorOutputDefinitionType,
@@ -37,7 +37,7 @@ type ApiEvaluatorRecordBase = {
version: number;
variables: string[];
// Read shape: an evaluator default can be incomplete or name experiment-only sources.
- mapping: PublicEvaluationRuleReadMappingType[] | null;
+ mapping: PromptVariableMappingReadType[] | null;
evaluationRuleCount: number;
createdAt: Date;
updatedAt: Date;
@@ -85,23 +85,23 @@ type ApiEvaluationRuleRecordBase = {
export type ApiWritableEvaluationRuleRecord = ApiEvaluationRuleRecordBase & {
evaluators: Array<{
evaluator: PublicEvaluationRuleEvaluatorType;
- mapping: PublicEvaluationRuleReadMappingType[] | null;
+ mapping: PromptVariableMappingReadType[] | null;
}>;
target: PublicEvaluationRuleTargetType;
filter: PublicEvaluationRuleFilterType[];
- mapping: PublicEvaluationRuleReadMappingType[];
+ mapping: PromptVariableMappingReadType[];
};
type ApiLegacyEvaluationRuleRecord = ApiEvaluationRuleRecordBase & {
evaluators: Array<{
evaluator: PublicEvaluationRuleEvaluatorType;
- mapping: LegacyEvaluationRuleMappingType[] | null;
+ mapping: LegacyPromptVariableMappingType[] | null;
}>;
target: PublicEvaluationRuleLegacyTargetType;
delay: number;
timeScope: JobTimeScope[];
filter: FilterCondition[];
- mapping: LegacyEvaluationRuleMappingType[];
+ mapping: LegacyPromptVariableMappingType[];
};
export type ApiEvaluationRuleRecord =
diff --git a/web/src/features/evals/server/unstable-public-api/validation.ts b/web/src/features/evals/server/unstable-public-api/validation.ts
index 2a01e00810..8cb2056dae 100644
--- a/web/src/features/evals/server/unstable-public-api/validation.ts
+++ b/web/src/features/evals/server/unstable-public-api/validation.ts
@@ -6,15 +6,15 @@ import { prisma } from "@langfuse/shared/src/db";
import { JSONPath } from "jsonpath-plus";
import {
EXPERIMENT_EVALUATION_RULE_FILTER_COLUMNS,
- ExperimentEvaluationRuleMappingSource,
+ ExperimentPromptVariableMappingSource,
OBSERVATION_EVALUATION_RULE_FILTER_COLUMNS,
- ObservationEvaluationRuleMappingSource,
+ ObservationPromptVariableMappingSource,
type PublicEvaluationRuleFilterType,
- type PublicEvaluationRuleMappingType,
+ type PromptVariableMappingInputType,
type PublicEvaluationRuleTargetType,
} from "@/src/features/public-api/types/unstable-public-evals-contract";
import { getEvaluatorDefinitionPreflightError } from "@/src/features/evals/server/evaluator-preflight";
-import { createUnstablePublicApiError } from "@/src/features/public-api/server/unstable-public-api-error-contract";
+import { createStructuredPublicApiError } from "@/src/features/public-api";
// Derived from the same column lists as the request schemas, so this runtime
// check cannot drift from the documented contract.
@@ -66,8 +66,8 @@ const SUPPORTED_FILTER_COLUMNS_BY_TARGET = {
// also guards paths that parse the target-agnostic mapping union) can never
// drift from the documented contract.
const SUPPORTED_MAPPING_SOURCES_BY_TARGET = {
- observation: new Set<string>(ObservationEvaluationRuleMappingSource.options),
- experiment: new Set<string>(ExperimentEvaluationRuleMappingSource.options),
+ observation: new Set<string>(ObservationPromptVariableMappingSource.options),
+ experiment: new Set<string>(ExperimentPromptVariableMappingSource.options),
} as const satisfies Record<PublicEvaluationRuleTargetType, Set<string>>;
export function validateEvaluationRuleFilters(params: {
@@ -79,7 +79,7 @@ export function validateEvaluationRuleFilters(params: {
for (const [filterIndex, filter] of params.filters.entries()) {
if (!supportedColumns.has(filter.column)) {
- throw createUnstablePublicApiError({
+ throw createStructuredPublicApiError({
httpCode: 400,
code: "invalid_filter_value",
message: `Filter column "${filter.column}" is not supported for target "${params.target}"`,
@@ -106,7 +106,7 @@ export function validateEvaluationRuleFilters(params: {
);
if (invalidValues.length > 0) {
- throw createUnstablePublicApiError({
+ throw createStructuredPublicApiError({
httpCode: 400,
code: "invalid_filter_value",
message: `Filter column "${filter.column}" contains unsupported value(s): ${invalidValues.join(", ")}`,
@@ -170,7 +170,7 @@ export async function assertEvaluationRuleFilterValuesExistForProject(params: {
);
if (invalidValues.length > 0) {
- throw createUnstablePublicApiError({
+ throw createStructuredPublicApiError({
httpCode: 400,
code: "invalid_filter_value",
message: `Filter column "datasetId" contains dataset id(s) that do not exist in this project: ${invalidValues.join(", ")}`,
@@ -192,7 +192,7 @@ function validateJsonPath(params: {
const { jsonPath, variable, mappingIndex } = params;
if (!jsonPath.startsWith("$")) {
- throw createUnstablePublicApiError({
+ throw createStructuredPublicApiError({
httpCode: 400,
code: "invalid_json_path",
message: `Mapping for variable "${variable}" has an invalid jsonPath "${jsonPath}". JSONPath expressions must start with "$".`,
@@ -241,7 +241,7 @@ function validateJsonPath(params: {
const expectedDelimiter = character === "]" ? "[" : "(";
if (delimiterStack.pop() !== expectedDelimiter) {
- throw createUnstablePublicApiError({
+ throw createStructuredPublicApiError({
httpCode: 400,
code: "invalid_json_path",
message: `Mapping for variable "${variable}" has an invalid jsonPath "${jsonPath}". JSONPath expressions must use balanced brackets and parentheses.`,
@@ -256,7 +256,7 @@ function validateJsonPath(params: {
}
if (openQuote || delimiterStack.length > 0) {
- throw createUnstablePublicApiError({
+ throw createStructuredPublicApiError({
httpCode: 400,
code: "invalid_json_path",
message: `Mapping for variable "${variable}" has an invalid jsonPath "${jsonPath}". JSONPath expressions must use balanced quotes, brackets, and parentheses.`,
@@ -276,7 +276,7 @@ function validateJsonPath(params: {
});
} catch (error) {
const message = error instanceof Error ? error.message : "Unknown error";
- throw createUnstablePublicApiError({
+ throw createStructuredPublicApiError({
httpCode: 400,
code: "invalid_json_path",
message: `Mapping for variable "${variable}" has an invalid jsonPath "${jsonPath}". ${message}`,
@@ -290,7 +290,7 @@ function validateJsonPath(params: {
}
export function validateEvaluatorVariableMappings(params: {
- mappings: PublicEvaluationRuleMappingType[];
+ mappings: PromptVariableMappingInputType[];
variables: string[];
target: PublicEvaluationRuleTargetType;
}) {
@@ -300,7 +300,7 @@ export function validateEvaluatorVariableMappings(params: {
for (const [mappingIndex, mapping] of params.mappings.entries()) {
if (!allowedSources.has(mapping.source)) {
- throw createUnstablePublicApiError({
+ throw createStructuredPublicApiError({
httpCode: 400,
code: "invalid_variable_mapping",
message: `Mapping source "${mapping.source}" is not supported for target "${params.target}"`,
@@ -313,7 +313,7 @@ export function validateEvaluatorVariableMappings(params: {
}
if (!variableSet.has(mapping.variable)) {
- throw createUnstablePublicApiError({
+ throw createStructuredPublicApiError({
httpCode: 400,
code: "invalid_variable_mapping",
message: `Mapping variable "${mapping.variable}" is not present in the evaluator prompt`,
@@ -325,7 +325,7 @@ export function validateEvaluatorVariableMappings(params: {
}
if (mappedVariables.has(mapping.variable)) {
- throw createUnstablePublicApiError({
+ throw createStructuredPublicApiError({
httpCode: 400,
code: "duplicate_variable_mapping",
message: `Mapping variable "${mapping.variable}" can only be mapped once`,
@@ -344,7 +344,7 @@ export function validateEvaluatorVariableMappings(params: {
);
if (missingVariables.length > 0) {
- throw createUnstablePublicApiError({
+ throw createStructuredPublicApiError({
httpCode: 400,
code: "missing_variable_mapping",
message: `Missing mappings for evaluator variables: ${missingVariables.join(", ")}`,
@@ -379,7 +379,7 @@ export async function assertEvaluatorDefinitionCanRunForPublicApi(params: {
const error = await getEvaluatorDefinitionPreflightError(params);
if (error) {
- throw createUnstablePublicApiError({
+ throw createStructuredPublicApiError({
httpCode: 422,
code: "evaluator_preflight_failed",
message: error,
diff --git a/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.stories.tsx b/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.stories.tsx
index 138d2090c7..cd358acc69 100644
--- a/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.stories.tsx
+++ b/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.stories.tsx
@@ -59,7 +59,6 @@ const managedTemplate = {
prompt: "Rate the relevance of {{generation}} to {{query}}.",
variables: [{ name: "query", defaultMapping: { field: "input" } }],
outputDefinition: {
- version: 2,
dataType: "NUMERIC",
score: { description: "Relevance." },
reasoning: { description: "One sentence." },
@@ -104,7 +103,6 @@ const sections: GallerySection[] = [
evaluator: {
...managedTemplate.evaluator,
outputDefinition: {
- version: 2,
dataType: "CATEGORICAL",
score: {
description: "Intent.",
@@ -124,7 +122,6 @@ const sections: GallerySection[] = [
evaluator: {
...managedTemplate.evaluator,
outputDefinition: {
- version: 2,
dataType: "BOOLEAN",
score: { description: "Out of scope." },
reasoning: { description: "One sentence." },
diff --git a/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGallerySection/EvaluatorGallerySection.stories.tsx b/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGallerySection/EvaluatorGallerySection.stories.tsx
index 3b3a9d1d10..53edae18ee 100644
--- a/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGallerySection/EvaluatorGallerySection.stories.tsx
+++ b/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGallerySection/EvaluatorGallerySection.stories.tsx
@@ -41,7 +41,6 @@ const template = {
prompt: "Rate the relevance of {{generation}} to {{query}}.",
variables: [{ name: "query", defaultMapping: { field: "input" } }],
outputDefinition: {
- version: 2,
dataType: "NUMERIC",
score: { description: "Relevance." },
reasoning: { description: "One sentence." },
diff --git a/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGallerySection/components/EvaluatorRecommendedCard/EvaluatorRecommendedCard.stories.tsx b/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGallerySection/components/EvaluatorRecommendedCard/EvaluatorRecommendedCard.stories.tsx
index 8685380bf4..1995978c73 100644
--- a/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGallerySection/components/EvaluatorRecommendedCard/EvaluatorRecommendedCard.stories.tsx
+++ b/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGallerySection/components/EvaluatorRecommendedCard/EvaluatorRecommendedCard.stories.tsx
@@ -18,7 +18,6 @@ const template = {
prompt: "Classify {{input}}.",
variables: [{ name: "input", defaultMapping: { field: "input" } }],
outputDefinition: {
- version: 2,
dataType: "CATEGORICAL",
score: {
description: "Intent.",
diff --git a/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGallerySection/components/EvaluatorTemplateRow/EvaluatorTemplateRow.stories.tsx b/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGallerySection/components/EvaluatorTemplateRow/EvaluatorTemplateRow.stories.tsx
index 46ac4975c2..c983af03f2 100644
--- a/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGallerySection/components/EvaluatorTemplateRow/EvaluatorTemplateRow.stories.tsx
+++ b/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGallerySection/components/EvaluatorTemplateRow/EvaluatorTemplateRow.stories.tsx
@@ -21,7 +21,6 @@ const managedTemplate = {
prompt: "Rate the relevance of {{generation}} to {{query}}.",
variables: [{ name: "query", defaultMapping: { field: "input" } }],
outputDefinition: {
- version: 2,
dataType: "NUMERIC",
score: { description: "Relevance." },
reasoning: { description: "One sentence." },
diff --git a/web/src/features/evals/v2/components/EvaluatorsEmptyState/components/EvaluatorsEmptyStateView/EvaluatorsEmptyStateView.stories.tsx b/web/src/features/evals/v2/components/EvaluatorsEmptyState/components/EvaluatorsEmptyStateView/EvaluatorsEmptyStateView.stories.tsx
index 5848847adf..c5db15b77b 100644
--- a/web/src/features/evals/v2/components/EvaluatorsEmptyState/components/EvaluatorsEmptyStateView/EvaluatorsEmptyStateView.stories.tsx
+++ b/web/src/features/evals/v2/components/EvaluatorsEmptyState/components/EvaluatorsEmptyStateView/EvaluatorsEmptyStateView.stories.tsx
@@ -22,7 +22,6 @@ const startingPoints = [
prompt: "Classify {{input}}.",
variables: [{ name: "input", defaultMapping: { field: "input" } }],
outputDefinition: {
- version: 2,
dataType: "CATEGORICAL",
score: {
description: "Topic.",
@@ -54,7 +53,6 @@ const startingPoints = [
{ name: "last_user_message", defaultMapping: { field: "input" } },
],
outputDefinition: {
- version: 2,
dataType: "BOOLEAN",
score: { description: "True if disagreement is present." },
reasoning: { description: "One sentence." },
diff --git a/web/src/features/evals/v2/constants/managedTemplatesCatalog.ts b/web/src/features/evals/v2/constants/managedTemplatesCatalog.ts
index 2d5e42f2f0..25afb94cd8 100644
--- a/web/src/features/evals/v2/constants/managedTemplatesCatalog.ts
+++ b/web/src/features/evals/v2/constants/managedTemplatesCatalog.ts
@@ -107,7 +107,6 @@ Replace these examples with your own taxonomy before use:
User message: {{input}}`,
variables: [{ name: "input", defaultMapping: { field: "input" } }],
outputDefinition: {
- version: 2,
dataType: "CATEGORICAL",
score: {
description: "Predicted category.",
@@ -193,7 +192,6 @@ Think step by step and return the structured result.`,
{ name: "last_user_message", defaultMapping: { field: "input" } },
],
outputDefinition: {
- version: 2,
dataType: "BOOLEAN",
score: {
description: "Boolean verdict.",
@@ -257,7 +255,6 @@ Last user message: {{last_user_message}}`,
{ name: "last_user_message", defaultMapping: { field: "input" } },
],
outputDefinition: {
- version: 2,
dataType: "BOOLEAN",
score: {
description: "Boolean verdict.",
@@ -412,7 +409,6 @@ Last user message: {{last_user_message}}`,
{ name: "last_user_message", defaultMapping: { field: "input" } },
],
outputDefinition: {
- version: 2,
dataType: "BOOLEAN",
score: {
description: "True if distress is present, false otherwise.",
@@ -477,7 +473,6 @@ Expected output: {{expected_output}}`,
},
],
outputDefinition: {
- version: 2,
dataType: "BOOLEAN",
score: {
description: "Boolean verdict.",
@@ -730,7 +725,6 @@ Assistant output: {{assistant_output}}`,
{ name: "assistant_output", defaultMapping: { field: "output" } },
],
outputDefinition: {
- version: 2,
dataType: "CATEGORICAL",
score: {
description: "Relevance label.",
@@ -782,7 +776,6 @@ Assistant output: {{assistant_output}}`,
{ name: "assistant_output", defaultMapping: { field: "output" } },
],
outputDefinition: {
- version: 2,
dataType: "BOOLEAN",
score: { description: "Boolean verdict." },
reasoning: {
@@ -834,7 +827,6 @@ Replace these example definitions with your own taxonomy before use:
Input: {{input}}`,
variables: [{ name: "input", defaultMapping: { field: "input" } }],
outputDefinition: {
- version: 2,
dataType: "CATEGORICAL",
score: {
description: "Predicted category label.",
@@ -881,7 +873,6 @@ You will receive an input and must assign exactly one primary language category.
Input: {{input}}`,
variables: [{ name: "input", defaultMapping: { field: "input" } }],
outputDefinition: {
- version: 2,
dataType: "CATEGORICAL",
score: {
description: "Predicted category.",
@@ -950,7 +941,6 @@ Context: {{context}}`,
{ name: "context", defaultMapping: { field: "input" } },
],
outputDefinition: {
- version: 2,
dataType: "CATEGORICAL",
score: {
description: "Groundedness label.",
@@ -1003,7 +993,6 @@ Context: {{context}}`,
{ name: "context", defaultMapping: { field: "input" } },
],
outputDefinition: {
- version: 2,
dataType: "CATEGORICAL",
score: {
description: "Context precision label.",
@@ -1062,7 +1051,6 @@ Context: {{context}}`,
{ name: "context", defaultMapping: { field: "input" } },
],
outputDefinition: {
- version: 2,
dataType: "CATEGORICAL",
score: {
description: "Context recall label.",
@@ -1127,7 +1115,6 @@ True if sensitive, identifying personal information appears in the output, false
Assistant output: {{output}}`,
variables: [{ name: "output", defaultMapping: { field: "output" } }],
outputDefinition: {
- version: 2,
dataType: "BOOLEAN",
score: {
description: "Boolean verdict.",
@@ -1174,7 +1161,6 @@ Assistant output: {{assistant_output}}`,
{ name: "assistant_output", defaultMapping: { field: "output" } },
],
outputDefinition: {
- version: 2,
dataType: "BOOLEAN",
score: { description: "Boolean verdict." },
reasoning: {
@@ -1225,7 +1211,6 @@ True if prompt injection is detected, false otherwise.
Input text: {{input_text}}`,
variables: [{ name: "input_text", defaultMapping: { field: "input" } }],
outputDefinition: {
- version: 2,
dataType: "BOOLEAN",
score: {
description: "Boolean verdict.",
@@ -1279,7 +1264,6 @@ Classify the user's primary requested outcome into exactly one engineering task
Task text: {{task_text}}`,
variables: [{ name: "task_text", defaultMapping: { field: "input" } }],
outputDefinition: {
- version: 2,
dataType: "CATEGORICAL",
score: {
description: "Engineering task-type label.",
@@ -1344,7 +1328,6 @@ Classify the task into the one department it most likely serves.
Task text: {{task_text}}`,
variables: [{ name: "task_text", defaultMapping: { field: "input" } }],
outputDefinition: {
- version: 2,
dataType: "CATEGORICAL",
score: {
description: "Department usage label.",
diff --git a/web/src/features/evals/v2/fns/evaluators/prepareEvaluatorDraft.clienttest.ts b/web/src/features/evals/v2/fns/evaluators/prepareEvaluatorDraft.clienttest.ts
index ec83f9f06e..0b374e90ac 100644
--- a/web/src/features/evals/v2/fns/evaluators/prepareEvaluatorDraft.clienttest.ts
+++ b/web/src/features/evals/v2/fns/evaluators/prepareEvaluatorDraft.clienttest.ts
@@ -14,7 +14,6 @@ describe("prepareEvaluatorDraft", () => {
vars: ["output"],
variableMapping: [],
outputDefinition: {
- version: 2 as const,
dataType: ScoreDataTypeEnum.NUMERIC,
score: { description: "Quality" },
reasoning: { description: "Reasoning" },
@@ -48,10 +47,11 @@ describe("prepareEvaluatorDraft", () => {
definition: {
type: "LLM_AS_JUDGE",
prompt: "Judge {{output}}",
- provider: "openai",
- model: "gpt-test",
- modelParams: { temperature: 0.2 },
- vars: ["output"],
+ modelConfig: {
+ provider: "openai",
+ model: "gpt-test",
+ modelParams: { temperature: 0.2 },
+ },
variableMapping: [
{
templateVariable: "output",
diff --git a/web/src/features/evals/v2/fns/evaluators/prepareEvaluatorDraft.ts b/web/src/features/evals/v2/fns/evaluators/prepareEvaluatorDraft.ts
index 97d91b2f02..20a5b8454a 100644
--- a/web/src/features/evals/v2/fns/evaluators/prepareEvaluatorDraft.ts
+++ b/web/src/features/evals/v2/fns/evaluators/prepareEvaluatorDraft.ts
@@ -25,24 +25,20 @@ export function prepareEvaluatorDraft(params: EvaluatorSetupDraftState) {
variableFields: params.variableFields,
})
: [];
- const variables = mappings.map(({ variable }) => variable);
const definition =
params.type === "LLM_AS_JUDGE"
? outputDefinition
? {
type: params.type,
prompt: params.prompt,
- provider:
- params.modelMode === "custom"
- ? (params.selectedModel?.provider ?? null)
+ modelConfig:
+ params.modelMode === "custom" && params.selectedModel
+ ? {
+ provider: params.selectedModel.provider,
+ model: params.selectedModel.model,
+ modelParams: params.modelParams,
+ }
: null,
- model:
- params.modelMode === "custom"
- ? (params.selectedModel?.model ?? null)
- : null,
- modelParams:
- params.modelMode === "custom" ? params.modelParams : null,
- vars: variables,
variableMapping: mappings.map(({ variable, fieldState }) => ({
templateVariable: variable,
selectedColumnId: fieldState.selectedColumnId,
diff --git a/web/src/features/evals/v2/fns/scoreOutput/buildScoreOutputDefinition.ts b/web/src/features/evals/v2/fns/scoreOutput/buildScoreOutputDefinition.ts
index a548eb1092..4f55872a0e 100644
--- a/web/src/features/evals/v2/fns/scoreOutput/buildScoreOutputDefinition.ts
+++ b/web/src/features/evals/v2/fns/scoreOutput/buildScoreOutputDefinition.ts
@@ -1,7 +1,7 @@
import {
- PersistedEvalOutputDefinitionSchema,
+ EvalOutputDefinitionSchema,
ScoreDataTypeEnum,
- type PersistedEvalOutputDefinition,
+ type EvalOutputDefinition,
} from "@langfuse/shared";
import type { ScoreOutputFormState } from "@/src/features/evals/v2/scoreOutputTypes";
import {
@@ -18,9 +18,8 @@ function parseNumericBound(key: "minValue" | "maxValue", raw: string) {
export function buildScoreOutputDefinition(
state: ScoreOutputFormState,
-): PersistedEvalOutputDefinition | null {
+): EvalOutputDefinition | null {
const base = {
- version: 2 as const,
dataType: state.dataType,
reasoning: {
description:
@@ -52,6 +51,6 @@ export function buildScoreOutputDefinition(
...base,
score: { description: scoreDescription },
};
- const parsed = PersistedEvalOutputDefinitionSchema.safeParse(candidate);
+ const parsed = EvalOutputDefinitionSchema.safeParse(candidate);
return parsed.success ? parsed.data : null;
}
diff --git a/web/src/features/evals/v2/fns/templateGallery/visibleGallerySections.clienttest.ts b/web/src/features/evals/v2/fns/templateGallery/visibleGallerySections.clienttest.ts
index ae8093333c..444e7d6cc3 100644
--- a/web/src/features/evals/v2/fns/templateGallery/visibleGallerySections.clienttest.ts
+++ b/web/src/features/evals/v2/fns/templateGallery/visibleGallerySections.clienttest.ts
@@ -22,7 +22,6 @@ const template = {
prompt: "Rate relevance.",
variables: [{ name: "query", defaultMapping: { field: "input" } }],
outputDefinition: {
- version: 2,
dataType: "NUMERIC",
score: { description: "Relevance." },
reasoning: { description: "One sentence." },
diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorErrors.ts b/web/src/features/evals/v2/server/evaluators/evaluatorErrors.ts
index 68d469e699..f7152cd46b 100644
--- a/web/src/features/evals/v2/server/evaluators/evaluatorErrors.ts
+++ b/web/src/features/evals/v2/server/evaluators/evaluatorErrors.ts
@@ -1,13 +1,8 @@
-import { BaseError } from "@langfuse/shared";
+import { BaseError, LangfuseConflictError } from "@langfuse/shared";
-export class EvaluatorVersionConflictError extends BaseError {
+export class EvaluatorVersionConflictError extends LangfuseConflictError {
constructor() {
- super(
- "EvaluatorVersionConflictError",
- 409,
- "Evaluator was updated concurrently. Retry the request.",
- true,
- );
+ super("Evaluator was updated concurrently. Retry the request.");
}
}
diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorRepository.ts b/web/src/features/evals/v2/server/evaluators/evaluatorRepository.ts
index 91782e64bd..04a172b887 100644
--- a/web/src/features/evals/v2/server/evaluators/evaluatorRepository.ts
+++ b/web/src/features/evals/v2/server/evaluators/evaluatorRepository.ts
@@ -27,15 +27,36 @@ import { creatorOptionsWhere, creatorWhere } from "../creatorFilterPrisma";
type PrismaTransaction = Prisma.TransactionClient;
+const createdByUser = {
+ select: { id: true, name: true, email: true },
+} as const;
+
const latestVersion = {
orderBy: { version: "desc" as const },
take: 1,
};
+const latestVersionWithCreator = {
+ ...latestVersion,
+ include: { createdByUser },
+};
+
const eventEvaluatorFilterColumnIds = new Set(
eventsEvalFilterColumns.map((column) => column.id),
);
+const publicEvaluationRuleAssignments = (projectId: string) =>
+ ({
+ where: { projectId },
+ orderBy: { createdAt: "desc" },
+ select: {
+ variableMapping: true,
+ evaluationRule: {
+ select: { id: true, targetObject: true },
+ },
+ },
+ }) satisfies Prisma.EvaluationRuleEvaluatorAssignmentFindManyArgs;
+
export const batchEligibleEvaluatorWhere = {
// Trace/dataset assignments carry rule-specific mappings that an
// observation batch cannot resolve when it addresses the evaluator alone.
@@ -264,8 +285,8 @@ export async function listEvaluators(params: {
skip: (params.page - 1) * params.limit,
take: params.limit,
include: {
- versions: latestVersion,
- createdByUser: { select: { name: true, email: true } },
+ versions: latestVersionWithCreator,
+ createdByUser,
_count: {
select: {
assignments: { where: { projectId: params.projectId } },
@@ -274,7 +295,7 @@ export async function listEvaluators(params: {
assignments: {
where: { projectId: params.projectId },
select: {
- evaluationRule: { select: { id: true, status: true } },
+ evaluationRule: { select: { id: true, name: true, status: true } },
},
},
},
@@ -288,6 +309,7 @@ export async function listEvaluators(params: {
return {
evaluators: evaluators.map(({ assignments, ...evaluator }) => ({
...evaluator,
+ assignments,
assignedRuleIds: assignments.map(
({ evaluationRule }) => evaluationRule.id,
),
@@ -303,6 +325,49 @@ export async function listEvaluators(params: {
};
}
+export async function listEvaluatorsCursor(params: {
+ prisma: PrismaClient;
+ projectId: string;
+ limit: number;
+ cursor?: { createdAt: Date; id: string };
+}) {
+ const baseWhere = await evaluatorWhere(params);
+ const where: Prisma.EvaluatorWhereInput = params.cursor
+ ? {
+ AND: [
+ baseWhere,
+ {
+ OR: [
+ { createdAt: { lt: params.cursor.createdAt } },
+ {
+ createdAt: params.cursor.createdAt,
+ id: { lt: params.cursor.id },
+ },
+ ],
+ },
+ ],
+ }
+ : baseWhere;
+ const records = await params.prisma.evaluator.findMany({
+ where,
+ orderBy: [{ createdAt: "desc" }, { id: "desc" }],
+ take: params.limit + 1,
+ include: {
+ versions: latestVersionWithCreator,
+ createdByUser,
+ assignments: publicEvaluationRuleAssignments(params.projectId),
+ },
+ });
+ const hasMore = records.length > params.limit;
+ const evaluators = records.slice(0, params.limit);
+ const last = evaluators.at(-1);
+ return {
+ evaluators,
+ nextCursor:
+ hasMore && last ? { createdAt: last.createdAt, id: last.id } : undefined,
+ };
+}
+
export async function listEvaluatorFilterOptions(params: {
prisma: PrismaClient;
projectId: string;
@@ -313,7 +378,7 @@ export async function listEvaluatorFilterOptions(params: {
select: {
name: true,
type: true,
- createdByUser: { select: { name: true, email: true } },
+ createdByUser,
versions: {
...latestVersion,
select: { model: true },
@@ -394,7 +459,7 @@ export async function listEvaluatorOptions(params: {
name: true,
type: true,
updatedAt: true,
- createdByUser: { select: { name: true, email: true } },
+ createdByUser,
blockedAt: true,
versions: {
orderBy: { version: "desc" },
@@ -423,7 +488,9 @@ export function findEvaluator(params: {
return params.prisma.evaluator.findFirst({
where: { id: params.evaluatorId, projectId: params.projectId },
include: {
- versions: latestVersion,
+ versions: latestVersionWithCreator,
+ createdByUser,
+ assignments: publicEvaluationRuleAssignments(params.projectId),
},
});
}
@@ -493,7 +560,8 @@ export async function listEvaluatorVersions(params: {
orderBy: { version: "desc" },
take: params.limit + 1,
include: {
- createdByUser: { select: { name: true, email: true } },
+ createdByUser,
+ evaluator: { select: { type: true } },
},
});
const hasMore = versions.length > params.limit;
@@ -547,7 +615,11 @@ export function createEvaluator(params: {
},
},
},
- include: { versions: latestVersion },
+ include: {
+ versions: latestVersionWithCreator,
+ createdByUser,
+ assignments: publicEvaluationRuleAssignments(params.input.projectId),
+ },
});
}
@@ -587,12 +659,17 @@ export function updateEvaluatorMetadata(params: {
tx: PrismaTransaction;
projectId: string;
evaluatorId: string;
- name: string;
- description: string | null;
+ name?: string;
+ description?: string | null;
}) {
return params.tx.evaluator.update({
where: { id: params.evaluatorId, projectId: params.projectId },
- data: { name: params.name, description: params.description },
+ data: {
+ ...(params.name === undefined ? {} : { name: params.name }),
+ ...(params.description === undefined
+ ? {}
+ : { description: params.description }),
+ },
});
}
diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorRouter.ts b/web/src/features/evals/v2/server/evaluators/evaluatorRouter.ts
index 5960449f24..9632a5afd1 100644
--- a/web/src/features/evals/v2/server/evaluators/evaluatorRouter.ts
+++ b/web/src/features/evals/v2/server/evaluators/evaluatorRouter.ts
@@ -10,7 +10,7 @@ import {
ActivationCostEstimatesSchema,
CreateEvaluatorSchema,
DeleteEvaluatorsSchema,
- EvaluatorDefinitionSchema,
+ EvaluatorDefinitionInputSchema,
EvaluatorIdSchema,
EvaluatorIdsSchema,
EvaluatorOptionsSchema,
@@ -26,7 +26,7 @@ import { getActivationCostEstimates } from "./activationCostService";
const TestEvaluatorSchema = z.object({
projectId: z.string(),
evaluatorId: z.string(),
- definition: EvaluatorDefinitionSchema,
+ definition: EvaluatorDefinitionInputSchema,
observationId: z.string(),
traceId: z.string(),
startTime: z.coerce.date(),
diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorService.ts b/web/src/features/evals/v2/server/evaluators/evaluatorService.ts
index f2b64b7ca6..c6087723b1 100644
--- a/web/src/features/evals/v2/server/evaluators/evaluatorService.ts
+++ b/web/src/features/evals/v2/server/evaluators/evaluatorService.ts
@@ -2,10 +2,12 @@ import { isDeepStrictEqual } from "node:util";
import { z } from "zod";
import {
EvalTemplateType,
+ type EvaluatorBlockReason,
type FilterState,
getBlockReasonForInvalidModelConfig,
getCodeEvalVariableMapping,
getEvaluatorBlockMetadata,
+ isEvaluatorBlockReasonRecoverableByDefinitionUpdate,
LangfuseConflictError,
LangfuseNotFoundError,
} from "@langfuse/shared";
@@ -29,6 +31,7 @@ import {
type EvaluatorDefinitionForPersistence,
type EvaluatorListOrderBy,
type EvaluatorVersionCursor,
+ type PatchEvaluatorInput,
type UpdateEvaluatorInput,
encodeEvaluatorVersionCursor,
EvaluatorDefinitionSchema,
@@ -97,6 +100,17 @@ export class EvaluatorService {
});
}
+ listCursor(params: {
+ projectId: string;
+ limit: number;
+ cursor?: { createdAt: Date; id: string };
+ }) {
+ return repository.listEvaluatorsCursor({
+ prisma: this.prisma,
+ ...params,
+ });
+ }
+
listFilterOptions(projectId: string) {
return repository.listEvaluatorFilterOptions({
prisma: this.prisma,
@@ -150,8 +164,8 @@ export class EvaluatorService {
...params,
cursor: params.cursor?.version,
});
- if (page.data.length === 0 && params.cursor === undefined) {
- throw new LangfuseNotFoundError("Evaluator not found");
+ if (page.data.length === 0) {
+ await this.get(params.projectId, params.evaluatorId);
}
return {
...page,
@@ -259,6 +273,7 @@ export class EvaluatorService {
}
throw error;
});
+ await invalidateProjectEvalConfigCaches(input.projectId);
await this.audit({
action: "create",
projectId: input.projectId,
@@ -324,6 +339,7 @@ export class EvaluatorService {
}),
};
});
+ await invalidateProjectEvalConfigCaches(input.projectId);
await this.audit({
action: result.action,
projectId: input.projectId,
@@ -347,6 +363,7 @@ export class EvaluatorService {
block,
}),
);
+ await invalidateProjectEvalConfigCaches(input.projectId);
await this.audit({
action: "update",
projectId: input.projectId,
@@ -355,6 +372,47 @@ export class EvaluatorService {
return evaluator;
}
+ async patch(input: PatchEvaluatorInput, createdByUserId: string | null) {
+ const validationResult = input.definition
+ ? await this.validatePatchedEvaluatorForPersistence(
+ input,
+ input.definition,
+ )
+ : null;
+ const evaluator = await this.prisma.$transaction((tx) =>
+ patchEvaluator({
+ tx,
+ input,
+ createdByUserId,
+ validationResult,
+ }),
+ );
+ await invalidateProjectEvalConfigCaches(input.projectId);
+ await this.audit({
+ action: "update",
+ projectId: input.projectId,
+ evaluatorId: evaluator.id,
+ });
+ return evaluator;
+ }
+
+ private async validatePatchedEvaluatorForPersistence(
+ input: PatchEvaluatorInput,
+ definition: EvaluatorDefinition,
+ ) {
+ const current = await this.get(input.projectId, input.evaluatorId);
+ return validateEvaluatorForPersistence({
+ projectId: input.projectId,
+ evaluatorId: input.evaluatorId,
+ name: input.name ?? current.name,
+ description:
+ input.description === undefined
+ ? current.description
+ : input.description,
+ definition,
+ });
+ }
+
async reactivate(params: { projectId: string; evaluatorId: string }) {
const { projectId, evaluatorId } = params;
const evaluator = await repository.findEvaluator({
@@ -561,6 +619,70 @@ async function deleteEvaluator(params: {
if (!deleted) throw new LangfuseNotFoundError("Evaluator not found");
}
+async function patchEvaluator(params: {
+ tx: Prisma.TransactionClient;
+ input: Omit<PatchEvaluatorInput, "definition"> & {
+ definition?: EvaluatorDefinition;
+ };
+ createdByUserId: string | null;
+ validationResult: Awaited<ReturnType<typeof validateEvaluatorForPersistence>>;
+}) {
+ const { tx, input, createdByUserId } = params;
+ const current = await repository.findEvaluator({
+ prisma: tx,
+ projectId: input.projectId,
+ evaluatorId: input.evaluatorId,
+ });
+ if (!current) throw new LangfuseNotFoundError("Evaluator not found");
+ if (input.definition && current.type !== input.definition.type) {
+ throw new LangfuseConflictError("Evaluator type cannot be changed");
+ }
+
+ if (input.name !== undefined || input.description !== undefined) {
+ await repository.updateEvaluatorMetadata({
+ tx,
+ projectId: input.projectId,
+ evaluatorId: input.evaluatorId,
+ name: input.name,
+ description: input.description,
+ });
+ }
+
+ if (input.definition) {
+ const latest = current.versions[0];
+ if (!latest) throw new LangfuseNotFoundError("Evaluator version not found");
+ const definitionChanged = !isDeepStrictEqual(
+ toEvaluatorDefinition(current.type, latest),
+ input.definition,
+ );
+ if (definitionChanged) {
+ await repository.appendEvaluatorVersion({
+ tx,
+ evaluatorId: input.evaluatorId,
+ version: latest.version + 1,
+ definition: prepareEvaluatorDefinitionForPersistence(input.definition),
+ createdByUserId,
+ });
+ }
+ await reconcileEvaluatorBlock({
+ tx,
+ projectId: input.projectId,
+ evaluatorId: input.evaluatorId,
+ validationResult: params.validationResult,
+ existingBlockedAt: current.blockedAt,
+ existingBlockReason: current.blockReason,
+ });
+ }
+
+ const updated = await repository.findEvaluator({
+ prisma: tx,
+ projectId: input.projectId,
+ evaluatorId: input.evaluatorId,
+ });
+ if (!updated) throw new LangfuseNotFoundError("Evaluator not found");
+ return updated;
+}
+
async function updateEvaluator(params: {
tx: Prisma.TransactionClient;
input: UpdateEvaluatorInput;
@@ -606,14 +728,14 @@ async function updateEvaluator(params: {
});
}
- if (params.block) {
- await repository.blockEvaluator({
- tx,
- projectId: input.projectId,
- evaluatorId: input.evaluatorId,
- ...params.block,
- });
- }
+ await reconcileEvaluatorBlock({
+ tx,
+ projectId: input.projectId,
+ evaluatorId: input.evaluatorId,
+ validationResult: params.block,
+ existingBlockedAt: current.blockedAt,
+ existingBlockReason: current.blockReason,
+ });
const updated = await repository.findEvaluator({
prisma: tx,
@@ -624,6 +746,34 @@ async function updateEvaluator(params: {
return updated;
}
+async function reconcileEvaluatorBlock(params: {
+ tx: Prisma.TransactionClient;
+ projectId: string;
+ evaluatorId: string;
+ validationResult: Awaited<ReturnType<typeof validateEvaluatorForPersistence>>;
+ existingBlockedAt: Date | null;
+ existingBlockReason: EvaluatorBlockReason | null;
+}) {
+ if (params.validationResult) {
+ await repository.blockEvaluator({
+ tx: params.tx,
+ projectId: params.projectId,
+ evaluatorId: params.evaluatorId,
+ ...params.validationResult,
+ });
+ return;
+ }
+ if (!params.existingBlockedAt) return;
+ if (
+ !isEvaluatorBlockReasonRecoverableByDefinitionUpdate(
+ params.existingBlockReason,
+ )
+ ) {
+ return;
+ }
+ await repository.unblockEvaluator(params);
+}
+
async function validateEvaluatorForPersistence(
input: CreateEvaluatorInput | UpdateEvaluatorInput,
) {
diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorTypes.ts b/web/src/features/evals/v2/server/evaluators/evaluatorTypes.ts
index 1f0a402b69..3e80c4f9bc 100644
--- a/web/src/features/evals/v2/server/evaluators/evaluatorTypes.ts
+++ b/web/src/features/evals/v2/server/evaluators/evaluatorTypes.ts
@@ -1,5 +1,7 @@
import {
+ EvalOutputDefinitionSchema,
EvalTemplateType,
+ extractVariables,
EvaluatorSourceCodeLanguage,
InvalidRequestError,
PersistedEvalOutputDefinitionSchema,
@@ -68,16 +70,50 @@ export const EvaluatorDefinitionSchema = z.discriminatedUnion("type", [
CodeEvaluatorDefinitionSchema,
]);
+export const EvaluatorModelConfigSchema = z.object({
+ provider: z.string().min(1),
+ model: z.string().min(1),
+ modelParams: ZodModelConfig.nullable().optional(),
+});
+
+const LlmEvaluatorDefinitionInputSchema = EvaluatorVersionBaseSchema.extend({
+ type: z.literal(EvalTemplateType.LLM_AS_JUDGE),
+ prompt: z.string().min(1),
+ modelConfig: EvaluatorModelConfigSchema.nullable(),
+ outputDefinition: EvalOutputDefinitionSchema,
+});
+
+export const EvaluatorDefinitionInputSchema = z
+ .discriminatedUnion("type", [
+ LlmEvaluatorDefinitionInputSchema,
+ CodeEvaluatorDefinitionSchema,
+ ])
+ .transform(
+ (definition): z.infer<typeof EvaluatorDefinitionSchema> =>
+ definition.type === EvalTemplateType.CODE
+ ? definition
+ : {
+ type: EvalTemplateType.LLM_AS_JUDGE,
+ prompt: definition.prompt,
+ provider: definition.modelConfig?.provider ?? null,
+ model: definition.modelConfig?.model ?? null,
+ modelParams: definition.modelConfig?.modelParams ?? null,
+ vars: extractVariables(definition.prompt),
+ variableMapping: definition.variableMapping,
+ outputDefinition: definition.outputDefinition,
+ },
+ );
+
export const CreateEvaluatorSchema = EvaluatorMetadataSchema.extend({
projectId: z.string(),
evaluatorId: z.uuid().optional(),
- definition: EvaluatorDefinitionSchema,
+ definition: EvaluatorDefinitionInputSchema,
});
export const UpdateEvaluatorSchema = EvaluatorMetadataSchema.extend({
projectId: z.string(),
evaluatorId: z.string(),
- definition: EvaluatorDefinitionSchema,
+ definition: EvaluatorDefinitionInputSchema,
});
export const EvaluatorIdSchema = z.object({
@@ -194,6 +230,11 @@ export type EvaluatorDefinitionForPersistence =
});
export type CreateEvaluatorInput = z.infer<typeof CreateEvaluatorSchema>;
export type UpdateEvaluatorInput = z.infer<typeof UpdateEvaluatorSchema>;
+export type PatchEvaluatorInput = Pick<
+ UpdateEvaluatorInput,
+ "projectId" | "evaluatorId"
+> &
+ Partial<Pick<UpdateEvaluatorInput, "name" | "description" | "definition">>;
export type DeleteEvaluatorsInput = z.infer<typeof DeleteEvaluatorsSchema>;
export type EvaluatorListOrderBy = z.infer<
typeof ListEvaluatorsSchema
diff --git a/web/src/features/evals/v2/server/rules/ruleErrors.ts b/web/src/features/evals/v2/server/rules/ruleErrors.ts
index 7903c8986f..35c8ce62a3 100644
--- a/web/src/features/evals/v2/server/rules/ruleErrors.ts
+++ b/web/src/features/evals/v2/server/rules/ruleErrors.ts
@@ -1,10 +1,29 @@
-import { LangfuseConflictError, EvalTargetObject } from "@langfuse/shared";
+import {
+ InvalidRequestError,
+ LangfuseConflictError,
+ EvalTargetObject,
+} from "@langfuse/shared";
import {
type PrismaClient,
type Prisma,
JobConfigState,
} from "@langfuse/shared/src/db";
+/**
+ * Enforced for every surface (tRPC, MCP, public API) through this shared guard,
+ * so the documented contract cannot drift per entry point.
+ */
+export function assertEnabledRuleHasAssignments(params: {
+ enabled: boolean;
+ assignmentCount: number;
+}) {
+ if (params.enabled && params.assignmentCount === 0) {
+ throw new InvalidRequestError(
+ "An enabled evaluation rule requires at least one evaluator assignment",
+ );
+ }
+}
+
/**
* Cap on simultaneously active evaluation rules per project. Enforced for every
* surface (tRPC, MCP, public API) through `assertActiveRuleLimitNotExceeded`, so
diff --git a/web/src/features/evals/v2/server/rules/ruleRepository.ts b/web/src/features/evals/v2/server/rules/ruleRepository.ts
index dbec658a76..e94611d8f1 100644
--- a/web/src/features/evals/v2/server/rules/ruleRepository.ts
+++ b/web/src/features/evals/v2/server/rules/ruleRepository.ts
@@ -50,7 +50,7 @@ const assignmentInclude = {
};
const ruleInclude = {
- createdByUser: { select: { name: true, email: true } },
+ createdByUser: { select: { id: true, name: true, email: true } },
assignments: assignmentInclude,
} satisfies Prisma.EvaluationRuleInclude;
@@ -153,6 +153,45 @@ export async function listRules(params: {
return { rules, totalItems };
}
+export async function listRulesCursor(params: {
+ prisma: PrismaClient;
+ input: Omit<ListRulesInput, "page"> & {
+ cursor?: { createdAt: Date; id: string };
+ };
+}) {
+ const baseWhere = ruleWhere(params.input);
+ const where: Prisma.EvaluationRuleWhereInput = params.input.cursor
+ ? {
+ AND: [
+ baseWhere,
+ {
+ OR: [
+ { createdAt: { lt: params.input.cursor.createdAt } },
+ {
+ createdAt: params.input.cursor.createdAt,
+ id: { lt: params.input.cursor.id },
+ },
+ ],
+ },
+ ],
+ }
+ : baseWhere;
+ const records = await params.prisma.evaluationRule.findMany({
+ where,
+ orderBy: [{ createdAt: "desc" }, { id: "desc" }],
+ take: params.input.limit + 1,
+ include: ruleInclude,
+ });
+ const hasMore = records.length > params.input.limit;
+ const rules = records.slice(0, params.input.limit);
+ const last = rules.at(-1);
+ return {
+ rules,
+ nextCursor:
+ hasMore && last ? { createdAt: last.createdAt, id: last.id } : undefined,
+ };
+}
+
export async function listReusableFilterCandidates(params: {
prisma: PrismaClient;
projectId: string;
@@ -195,7 +234,7 @@ export async function listRuleFilterOptions(params: {
where: { projectId: params.projectId },
select: {
name: true,
- createdByUser: { select: { name: true, email: true } },
+ createdByUser: { select: { id: true, name: true, email: true } },
},
}),
params.prisma.evaluationRule.findFirst({
@@ -349,15 +388,34 @@ export async function deleteRule(params: {
projectId: string;
ruleId: string;
}) {
+ const result = await params.prisma.evaluationRule.deleteMany({
+ where: { id: params.ruleId, projectId: params.projectId },
+ });
+ if (result.count === 0) return false;
+
// Executions carry no foreign key to the rule, so they have to go explicitly
// or they keep showing up in the eval log of a deleted rule.
await params.prisma.jobExecution.deleteMany({
where: { projectId: params.projectId, jobConfigurationId: params.ruleId },
});
+ return true;
+}
+
+export async function deleteRules(params: {
+ prisma: RulePrisma;
+ projectId: string;
+ ruleIds: string[];
+}) {
const result = await params.prisma.evaluationRule.deleteMany({
- where: { id: params.ruleId, projectId: params.projectId },
+ where: { projectId: params.projectId, id: { in: params.ruleIds } },
});
- return result.count > 0;
+ await params.prisma.jobExecution.deleteMany({
+ where: {
+ projectId: params.projectId,
+ jobConfigurationId: { in: params.ruleIds },
+ },
+ });
+ return result.count;
}
export async function listSelectedRuleIds(params: {
diff --git a/web/src/features/evals/v2/server/rules/ruleService.ts b/web/src/features/evals/v2/server/rules/ruleService.ts
index 9dc0912003..253b95f374 100644
--- a/web/src/features/evals/v2/server/rules/ruleService.ts
+++ b/web/src/features/evals/v2/server/rules/ruleService.ts
@@ -35,7 +35,10 @@ import type {
UpdateRuleInput,
} from "./ruleTypes";
import * as evaluatorRepository from "../evaluators/evaluatorRepository";
-import { assertActiveRuleLimitNotExceeded } from "./ruleErrors";
+import {
+ assertActiveRuleLimitNotExceeded,
+ assertEnabledRuleHasAssignments,
+} from "./ruleErrors";
import * as repository from "./ruleRepository";
import { isLegacyEvalTarget } from "@/src/features/evals/utils/typeHelpers";
import { prepareModernRuleVariableMapping } from "@/src/features/evals/v2/fns/variableMapping/prepareModernRuleVariableMapping";
@@ -68,6 +71,18 @@ export class RuleService {
return { rules: rules.map(toRuleResponse), totalItems };
}
+ async listCursor(
+ input: Omit<ListRulesInput, "page"> & {
+ cursor?: { createdAt: Date; id: string };
+ },
+ ) {
+ const { rules, nextCursor } = await repository.listRulesCursor({
+ prisma: this.prisma,
+ input,
+ });
+ return { rules: rules.map(toRuleResponse), nextCursor };
+ }
+
listFilterOptions(projectId: string) {
return repository.listRuleFilterOptions({
prisma: this.prisma,
@@ -245,6 +260,10 @@ export class RuleService {
normalized.filter,
);
this.assertUniqueAssignments(input.evaluatorAssignments);
+ assertEnabledRuleHasAssignments({
+ enabled: input.enabled,
+ assignmentCount: input.evaluatorAssignments.length,
+ });
const rule = await this.prisma.$transaction(async (prisma) => {
await assertActiveRuleLimitNotExceeded({
prisma,
@@ -356,6 +375,16 @@ export class RuleService {
input.ruleId,
);
this.assertLegacyRuleUpdateAllowed(current.targetObject, input);
+ const resultingAssignmentCount =
+ input.evaluatorMappings?.length ?? current.assignments.length;
+ assertEnabledRuleHasAssignments({
+ enabled:
+ input.enabled ??
+ (input.evaluatorMappings?.length === 0
+ ? false
+ : current.status === JobConfigState.ACTIVE),
+ assignmentCount: resultingAssignmentCount,
+ });
const currentTargetObject = current.targetObject as EvalTargetObject;
const currentFilter = current.filter as FilterState;
// Whether a rule targets experiments is derived from its filter, so an
@@ -375,10 +404,9 @@ export class RuleService {
targetObject: input.targetObject ?? inheritedTargetObject,
filter: effectiveFilter,
});
- const filter = this.validateRuleFilters(
- normalized.targetObject,
- normalized.filter,
- );
+ const filter = isLegacyEvalTarget(currentTargetObject)
+ ? currentFilter
+ : this.validateRuleFilters(normalized.targetObject, normalized.filter);
await assertActiveRuleLimitNotExceeded({
prisma,
projectId: input.projectId,
@@ -441,6 +469,10 @@ export class RuleService {
params.ruleId,
);
this.assertLegacyRuleCanBeEnabled(current.targetObject, params.enabled);
+ assertEnabledRuleHasAssignments({
+ enabled: params.enabled,
+ assignmentCount: current.assignments.length,
+ });
await assertActiveRuleLimitNotExceeded({
prisma,
projectId: params.projectId,
@@ -485,8 +517,10 @@ export class RuleService {
async deleteMany(input: RuleSelectionInput) {
const ruleIds = await this.prisma.$transaction(async (prisma) => {
const ids = await repository.listSelectedRuleIds({ prisma, input });
- await prisma.evaluationRule.deleteMany({
- where: { projectId: input.projectId, id: { in: ids } },
+ await repository.deleteRules({
+ prisma,
+ projectId: input.projectId,
+ ruleIds: ids,
});
return ids;
});
@@ -512,6 +546,7 @@ export class RuleService {
projectId: input.projectId,
isBatchAction: true,
search: input.search,
+ filter: input.filter,
};
const ids = await repository.listSelectedRuleIds({
prisma,
@@ -530,6 +565,18 @@ export class RuleService {
this.assertLegacyRuleCanBeEnabled("trace", input.enabled);
}
if (input.enabled) {
+ const unassignedRule = await prisma.evaluationRule.findFirst({
+ where: {
+ projectId: input.projectId,
+ id: { in: ids },
+ assignments: { none: {} },
+ },
+ select: { id: true },
+ });
+ assertEnabledRuleHasAssignments({
+ enabled: true,
+ assignmentCount: unassignedRule ? 0 : 1,
+ });
// Only rules that are not already active consume a new slot.
const newlyActivated = await prisma.evaluationRule.count({
where: {
diff --git a/web/src/features/evals/v2/server/rules/ruleTypes.ts b/web/src/features/evals/v2/server/rules/ruleTypes.ts
index a687913cd3..c11860b057 100644
--- a/web/src/features/evals/v2/server/rules/ruleTypes.ts
+++ b/web/src/features/evals/v2/server/rules/ruleTypes.ts
@@ -70,7 +70,10 @@ export const CreateRuleSchema = RuleMetadataSchema.extend({
projectId: z.string(),
targetObject: z
.enum([EvalTargetObject.EVENT, EvalTargetObject.EXPERIMENT])
- .default(EvalTargetObject.EVENT),
+ .default(EvalTargetObject.EVENT)
+ .describe(
+ "Deprecated: modern rules are event rules and experiment scope is expressed through filters.",
+ ),
enabled: z.boolean(),
evaluatorAssignments: z.array(RuleAssignmentInputSchema).max(100),
});
@@ -115,7 +118,6 @@ const FilteredRuleSelectionSchema = z.object({
projectId: z.string(),
isBatchAction: z.literal(true),
search: z.string().trim().max(200).optional(),
- enabled: z.boolean().optional(),
filter: ListRulesSchema.shape.filter,
});
diff --git a/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.clienttest.ts b/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.clienttest.ts
index 1d0b571d07..54a910dd75 100644
--- a/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.clienttest.ts
+++ b/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.clienttest.ts
@@ -260,7 +260,6 @@ describe("createEvaluatorSetupStore", () => {
},
],
outputDefinition: {
- version: 2,
dataType: "NUMERIC",
score: {
description: "Answer quality",
diff --git a/web/src/features/mcp/core/run-mcp-tool.ts b/web/src/features/mcp/core/run-mcp-tool.ts
index d14d4d7c52..056b145e6d 100644
--- a/web/src/features/mcp/core/run-mcp-tool.ts
+++ b/web/src/features/mcp/core/run-mcp-tool.ts
@@ -3,7 +3,7 @@ import { addUserToSpan, instrumentAsync } from "@langfuse/shared/src/server";
import { SpanKind, type Span } from "@opentelemetry/api";
import { ZodError } from "zod";
-import { UnstablePublicApiError } from "@/src/features/public-api/server/unstable-public-api-error-contract";
+import { StructuredPublicApiError } from "@/src/features/public-api";
import { isUserInputError } from "./errors";
import type { ServerContext } from "../types";
@@ -66,7 +66,7 @@ export const runMcpTool = async <TResult>({
if (error instanceof BaseError) {
span.setAttribute("mcp.error.http_code", error.httpCode);
}
- if (error instanceof UnstablePublicApiError) {
+ if (error instanceof StructuredPublicApiError) {
span.setAttribute("mcp.error.code", error.code);
}
throw error;
diff --git a/web/src/features/mcp/features/dashboardWidgets/tools/createDashboardWidget.ts b/web/src/features/mcp/features/dashboardWidgets/tools/createDashboardWidget.ts
index f8b454025b..ab0e359b04 100644
--- a/web/src/features/mcp/features/dashboardWidgets/tools/createDashboardWidget.ts
+++ b/web/src/features/mcp/features/dashboardWidgets/tools/createDashboardWidget.ts
@@ -12,7 +12,7 @@ import {
PostUnstableDashboardWidgetBody,
PostUnstableDashboardWidgetView,
} from "@/src/features/public-api/types/unstable-dashboard-widgets";
-import { UnstablePublicApiError } from "@/src/features/public-api/server/unstable-public-api-error-contract";
+import { StructuredPublicApiError } from "@/src/features/public-api";
import { buildDashboardWidgetUrl } from "@langfuse/shared/src/server";
export const DashboardWidgetFilterBaseSchema = z
@@ -76,7 +76,7 @@ const throwActionableDashboardWidgetError = (
error: unknown,
input: z.infer<typeof PostUnstableDashboardWidgetBody>,
): never => {
- if (!(error instanceof UnstablePublicApiError)) throw error;
+ if (!(error instanceof StructuredPublicApiError)) throw error;
const field = error.details?.field;
const allowedValues =
diff --git a/web/src/features/mcp/features/evals/rule-schema.ts b/web/src/features/mcp/features/evals/rule-schema.ts
index b72cafeb0b..4d13fc117b 100644
--- a/web/src/features/mcp/features/evals/rule-schema.ts
+++ b/web/src/features/mcp/features/evals/rule-schema.ts
@@ -9,8 +9,8 @@ import {
UpdateRuleSchema,
} from "@/src/features/evals/v2/server/rules/ruleTypes";
import {
- ObservationEvaluationRuleMapping,
- PublicEvaluationRuleReadMapping,
+ ObservationPromptVariableMappingInput,
+ PromptVariableMappingRead,
PublicEvaluatorType,
} from "@/src/features/public-api/types/unstable-public-evals-contract";
import { McpAdvancedFilterBaseSchema } from "../../core/filter-schema";
@@ -31,7 +31,7 @@ const EvaluationRuleEvaluatorInputBase = {
// The tool boundary speaks the public mapping contract (`variable`/`source`),
// not the stored column ids. Handlers translate via `toStoredMappingList`.
const VariableMappingSchema = z
- .array(ObservationEvaluationRuleMapping)
+ .array(ObservationPromptVariableMappingInput)
.describe(
"Optional rule-specific variable mapping for LLM evaluators. Omit for code evaluators, whose mapping is managed by Langfuse.",
);
@@ -159,7 +159,7 @@ export const EvaluationRuleResponseSchema = z
evaluatorId: z.string(),
evaluatorName: z.string(),
evaluatorType: PublicEvaluatorType,
- variableMapping: z.array(PublicEvaluationRuleReadMapping).nullable(),
+ variableMapping: z.array(PromptVariableMappingRead).nullable(),
}),
),
createdAt: z.date(),
diff --git a/web/src/features/mcp/features/evals/tools/createEvaluator.ts b/web/src/features/mcp/features/evals/tools/createEvaluator.ts
index 063ea460bf..40ba78fdb4 100644
--- a/web/src/features/mcp/features/evals/tools/createEvaluator.ts
+++ b/web/src/features/mcp/features/evals/tools/createEvaluator.ts
@@ -12,7 +12,7 @@ export const [createEvaluatorTool, handleCreateEvaluator] = defineTool({
name: "createEvaluator",
description: [
"Create a new evaluator with a stable id. Names do not act as identity and may be reused.",
- "Set type to `LLM_AS_JUDGE` and provide prompt + outputDefinition (provider, model, and modelParams are optional), or set type to `CODE` and provide sourceCode + sourceCodeLanguage.",
+ "Set type to `LLM_AS_JUDGE` and provide prompt + outputDefinition. Omit modelConfig to use the project default, or provide modelConfig with provider, model, and optional modelParams. For `CODE`, provide sourceCode + sourceCodeLanguage.",
"Use updateEvaluator with the returned evaluatorId to update it or append a new immutable definition version.",
].join(" "),
baseSchema: McpEvaluatorInputBase,
diff --git a/web/src/features/mcp/features/evals/tools/evaluatorInput.ts b/web/src/features/mcp/features/evals/tools/evaluatorInput.ts
index 1c47aa6cd0..af1472953a 100644
--- a/web/src/features/mcp/features/evals/tools/evaluatorInput.ts
+++ b/web/src/features/mcp/features/evals/tools/evaluatorInput.ts
@@ -1,34 +1,46 @@
-import { EvalTemplateType, extractVariables } from "@langfuse/shared";
+import { EvalTemplateType, observationVariableMapping } from "@langfuse/shared";
import { z } from "zod";
import {
CodeEvaluatorDefinitionSchema,
CreateEvaluatorSchema,
+ EvaluatorModelConfigSchema,
LlmEvaluatorDefinitionSchema,
} from "@/src/features/evals/v2/server/evaluators/evaluatorTypes";
const CreateEvaluatorWithoutProjectSchema = CreateEvaluatorSchema.omit({
projectId: true,
});
+
+const McpEvaluatorModelConfigSchema = EvaluatorModelConfigSchema.extend({
+ modelParams: z
+ .object({
+ max_tokens: z.number().optional(),
+ temperature: z.number().optional(),
+ top_p: z.number().optional(),
+ maxReasoningTokens: z.number().optional(),
+ providerOptions: z.record(z.string(), z.any()).optional(),
+ })
+ .optional(),
+});
+
+const McpObservationVariableMappingSchema = observationVariableMapping.extend({
+ jsonSelector: z.string().optional(),
+});
+
export const McpEvaluatorInputBase = z.object({
name: CreateEvaluatorSchema.shape.name,
description: CreateEvaluatorSchema.shape.description.unwrap().optional(),
type: z.enum(EvalTemplateType),
prompt: LlmEvaluatorDefinitionSchema.shape.prompt.optional(),
- provider: LlmEvaluatorDefinitionSchema.shape.provider.unwrap().optional(),
- model: LlmEvaluatorDefinitionSchema.shape.model.unwrap().optional(),
- modelParams: z.record(z.string(), z.unknown()).optional(),
+ modelConfig: McpEvaluatorModelConfigSchema.optional().describe(
+ "Optional custom model configuration. Omit to use the project default model.",
+ ),
outputDefinition: z.record(z.string(), z.unknown()).optional(),
sourceCode: CodeEvaluatorDefinitionSchema.shape.sourceCode.optional(),
sourceCodeLanguage:
CodeEvaluatorDefinitionSchema.shape.sourceCodeLanguage.optional(),
variableMapping: z
- .array(
- z.object({
- templateVariable: z.string(),
- selectedColumnId: z.string(),
- jsonSelector: z.string().optional(),
- }),
- )
+ .array(McpObservationVariableMappingSchema)
.optional()
.describe("Variable mappings for LLM-as-a-judge evaluators only."),
});
@@ -41,10 +53,7 @@ function toEvaluatorInput(input: z.infer<typeof McpEvaluatorInputBase>) {
definition: {
type: input.type,
prompt: input.prompt!,
- provider: input.provider ?? null,
- model: input.model ?? null,
- modelParams: input.modelParams ?? null,
- vars: extractVariables(input.prompt!),
+ modelConfig: input.modelConfig ?? null,
variableMapping: input.variableMapping ?? null,
outputDefinition: input.outputDefinition,
},
diff --git a/web/src/features/mcp/features/evals/tools/updateEvaluator.ts b/web/src/features/mcp/features/evals/tools/updateEvaluator.ts
index a658b11cbc..8de8f78d49 100644
--- a/web/src/features/mcp/features/evals/tools/updateEvaluator.ts
+++ b/web/src/features/mcp/features/evals/tools/updateEvaluator.ts
@@ -12,7 +12,7 @@ export const [updateEvaluatorTool, handleUpdateEvaluator] = defineTool({
name: "updateEvaluator",
description: [
"Update an evaluator by stable id. Definition changes append an immutable version; name and description changes do not.",
- "Set type to `LLM_AS_JUDGE` and provide prompt + outputDefinition (provider, model, and modelParams are optional), or set type to `CODE` and provide sourceCode + sourceCodeLanguage.",
+ "Set type to `LLM_AS_JUDGE` and provide prompt + outputDefinition. Omit modelConfig to use the project default, or provide modelConfig with provider, model, and optional modelParams. For `CODE`, provide sourceCode + sourceCodeLanguage.",
].join(" "),
baseSchema: McpUpdateEvaluatorInputBase,
inputSchema: McpUpdateEvaluatorInput,
diff --git a/web/src/features/public-api/index.ts b/web/src/features/public-api/index.ts
new file mode 100644
index 0000000000..4f2daa4426
--- /dev/null
+++ b/web/src/features/public-api/index.ts
@@ -0,0 +1,17 @@
+export {
+ DeleteEvaluatorResponse,
+ Evaluator,
+ ListEvaluatorVersionsResponse,
+ ListEvaluatorsResponse,
+ LlmAsJudgeEvaluator,
+} from "./types/evaluation/evaluators";
+export {
+ DeleteEvaluationRuleResponse,
+ EvaluationRule,
+ ListEvaluationRulesResponse,
+} from "./types/evaluation/evaluationRules";
+export { PublicApiError } from "./types/public-api-errors";
+export {
+ createStructuredPublicApiError,
+ StructuredPublicApiError,
+} from "./types/structuredPublicApiError";
diff --git a/web/src/features/public-api/server/RateLimitService.ts b/web/src/features/public-api/server/RateLimitService.ts
index ad14555f3e..bfbe86650b 100644
--- a/web/src/features/public-api/server/RateLimitService.ts
+++ b/web/src/features/public-api/server/RateLimitService.ts
@@ -18,10 +18,10 @@ import {
import { env as sharedEnv } from "@langfuse/shared/src/env";
import { type NextApiResponse } from "next";
import {
- createUnstablePublicApiRateLimitError,
- sendUnstablePublicApiErrorResponse,
+ createStructuredPublicApiRateLimitError,
+ sendStructuredPublicApiErrorResponse,
type PublicApiErrorContract,
-} from "@/src/features/public-api/server/unstable-public-api-error-contract";
+} from "./structuredPublicApiErrorContract";
import { type RateLimitUpgradePath } from "@/src/features/public-api/server/rateLimitUpgradePaths";
export const RATE_LIMIT_REDIS_KEY_PREFIX = "rate-limit";
@@ -217,9 +217,9 @@ export const sendRateLimitResponse = (
res.setHeader(header, value);
}
- return sendUnstablePublicApiErrorResponse(
+ return sendStructuredPublicApiErrorResponse(
res,
- createUnstablePublicApiRateLimitError(rateLimitRes, responseOptions),
+ createStructuredPublicApiRateLimitError(rateLimitRes, responseOptions),
);
};
diff --git a/web/src/features/public-api/server/createAuthedProjectAPIRoute.ts b/web/src/features/public-api/server/createAuthedProjectAPIRoute.ts
index 6af52dd881..5d1261932c 100644
--- a/web/src/features/public-api/server/createAuthedProjectAPIRoute.ts
+++ b/web/src/features/public-api/server/createAuthedProjectAPIRoute.ts
@@ -23,12 +23,12 @@ import { env } from "@/src/env.mjs";
import { isZodError } from "@/src/features/public-api/server/withMiddlewares";
import { isPrismaException } from "@/src/utils/exceptions";
import {
- createUnstablePublicApiAuthError,
- createUnstablePublicApiRequestValidationError,
- sendUnstablePublicApiErrorResponse,
- unstablePublicEvalsErrorContract,
+ createStructuredPublicApiAuthError,
+ createStructuredPublicApiRequestValidationError,
+ sendStructuredPublicApiErrorResponse,
+ structuredPublicApiErrorContract,
type PublicApiErrorContract,
-} from "@/src/features/public-api/server/unstable-public-api-error-contract";
+} from "./structuredPublicApiErrorContract";
import { clickHouseRouteForRequest } from "@/src/features/public-api/server/clickHouseRequestTags";
import { attachDeprecation } from "@/src/features/public-api/server/deprecations";
@@ -353,10 +353,10 @@ export const createAuthedProjectAPIRoute = <
if (isPrismaException(error)) {
traceException(error);
- if (routeConfig.errorContract === unstablePublicEvalsErrorContract) {
- return sendUnstablePublicApiErrorResponse(
+ if (routeConfig.errorContract === structuredPublicApiErrorContract) {
+ return sendStructuredPublicApiErrorResponse(
res,
- createUnstablePublicApiAuthError({
+ createStructuredPublicApiAuthError({
statusCode: 503,
message: "Service Unavailable",
}),
@@ -370,10 +370,10 @@ export const createAuthedProjectAPIRoute = <
const statusCode = error.status ?? 401;
const message = error.message ?? "Authentication failed";
- if (routeConfig.errorContract === unstablePublicEvalsErrorContract) {
- return sendUnstablePublicApiErrorResponse(
+ if (routeConfig.errorContract === structuredPublicApiErrorContract) {
+ return sendStructuredPublicApiErrorResponse(
res,
- createUnstablePublicApiAuthError({ statusCode, message }),
+ createStructuredPublicApiAuthError({ statusCode, message }),
);
}
@@ -406,12 +406,12 @@ export const createAuthedProjectAPIRoute = <
: ({} as z.infer<TQuery>);
} catch (error) {
if (
- routeConfig.errorContract === unstablePublicEvalsErrorContract &&
+ routeConfig.errorContract === structuredPublicApiErrorContract &&
isZodError(error)
) {
- return sendUnstablePublicApiErrorResponse(
+ return sendStructuredPublicApiErrorResponse(
res,
- createUnstablePublicApiRequestValidationError({
+ createStructuredPublicApiRequestValidationError({
error,
requestPart: "query",
}),
@@ -428,12 +428,12 @@ export const createAuthedProjectAPIRoute = <
: ({} as z.infer<TBody>);
} catch (error) {
if (
- routeConfig.errorContract === unstablePublicEvalsErrorContract &&
+ routeConfig.errorContract === structuredPublicApiErrorContract &&
isZodError(error)
) {
- return sendUnstablePublicApiErrorResponse(
+ return sendStructuredPublicApiErrorResponse(
res,
- createUnstablePublicApiRequestValidationError({
+ createStructuredPublicApiRequestValidationError({
error,
requestPart: "body",
}),
diff --git a/web/src/features/public-api/server/evaluation/evaluationAdapters.ts b/web/src/features/public-api/server/evaluation/evaluationAdapters.ts
new file mode 100644
index 0000000000..6dc59e7229
--- /dev/null
+++ b/web/src/features/public-api/server/evaluation/evaluationAdapters.ts
@@ -0,0 +1,397 @@
+import {
+ EvalTemplateType,
+ extractVariables,
+ InternalServerError,
+ observationVariableMappingList,
+ PersistedEvalOutputDefinitionSchema,
+ resolvePersistedEvalOutputDefinition,
+ variableMappingList,
+ type FilterState,
+ type ObservationVariableMapping,
+} from "@langfuse/shared";
+import { logger } from "@langfuse/shared/src/server";
+import {
+ EvaluatorDefinitionInputSchema,
+ isLegacyEvalTarget,
+ type EvaluatorService,
+ type RuleService,
+} from "@/src/features/evals/server";
+import {
+ EvaluationRule,
+ type CreateEvaluationRuleBodyType,
+} from "@/src/features/public-api/types/evaluation/evaluationRules";
+import {
+ Evaluator,
+ EvaluatorVersion,
+ type EvaluatorDefinitionType,
+} from "@/src/features/public-api/types/evaluation/evaluators";
+import {
+ LegacyPromptVariableMapping,
+ PublicEvaluationRuleReadFilter,
+ PUBLIC_EVALUATOR_TYPE_CODE,
+ PUBLIC_EVALUATOR_TYPE_LLM_AS_JUDGE,
+ type LegacyPromptVariableMappingType,
+ type PromptVariableMappingInputType,
+ type PromptVariableMappingReadType,
+ type PublicEvaluatorOutputDefinitionType,
+} from "@/src/features/public-api/types/evaluation/publicEvalsContract";
+
+const PUBLIC_MAPPING_SOURCE_TO_INTERNAL_COLUMN: Record<
+ PromptVariableMappingInputType["source"],
+ ObservationVariableMapping["selectedColumnId"]
+> = {
+ input: "input",
+ output: "output",
+ metadata: "metadata",
+ tool_calls: "toolCalls",
+ expected_output: "experimentItemExpectedOutput",
+ experiment_item_metadata: "experimentItemMetadata",
+};
+
+const INTERNAL_MAPPING_COLUMN_TO_PUBLIC_SOURCE: Record<
+ string,
+ PromptVariableMappingInputType["source"]
+> = {
+ input: "input",
+ output: "output",
+ metadata: "metadata",
+ toolCalls: "tool_calls",
+ expected_output: "expected_output",
+ expectedOutput: "expected_output",
+ experiment_item_expected_output: "expected_output",
+ experimentItemExpectedOutput: "expected_output",
+ experimentItemMetadata: "experiment_item_metadata",
+ experiment_item_metadata: "experiment_item_metadata",
+};
+
+function toStoredMappingList(mappings: PromptVariableMappingInputType[]) {
+ return observationVariableMappingList.parse(
+ mappings.map((mapping) => ({
+ templateVariable: mapping.variable,
+ selectedColumnId:
+ PUBLIC_MAPPING_SOURCE_TO_INTERNAL_COLUMN[mapping.source],
+ jsonSelector: mapping.jsonPath ?? null,
+ })),
+ );
+}
+
+function toApiReadMappings(mappings: unknown): PromptVariableMappingReadType[] {
+ const parsed = observationVariableMappingList.safeParse(mappings);
+ if (!parsed.success) {
+ logger.error("Failed to parse public evaluation rule mappings", {
+ issues: parsed.error.issues,
+ });
+ throw new InternalServerError("Evaluation rule mapping is corrupted");
+ }
+
+ return parsed.data.map((mapping) => {
+ const source =
+ INTERNAL_MAPPING_COLUMN_TO_PUBLIC_SOURCE[mapping.selectedColumnId];
+ if (!source && mapping.selectedColumnId.trim()) {
+ throw new InternalServerError("Evaluation rule mapping is corrupted");
+ }
+ return {
+ variable: mapping.templateVariable,
+ source: source ?? null,
+ ...(mapping.jsonSelector ? { jsonPath: mapping.jsonSelector } : {}),
+ };
+ });
+}
+
+function toApiLegacyMappings(
+ mappings: unknown,
+): LegacyPromptVariableMappingType[] {
+ const parsed = variableMappingList.safeParse(mappings);
+ if (!parsed.success) {
+ logger.error("Failed to parse public legacy evaluation rule mappings", {
+ issues: parsed.error.issues,
+ });
+ throw new InternalServerError("Evaluation rule mapping is corrupted");
+ }
+
+ return LegacyPromptVariableMapping.array().parse(
+ parsed.data.map((mapping) => ({
+ mappingType: "legacy",
+ variable: mapping.templateVariable,
+ langfuseObject: mapping.langfuseObject,
+ objectName: mapping.objectName ?? null,
+ source: mapping.selectedColumnId,
+ ...(mapping.jsonSelector ? { jsonPath: mapping.jsonSelector } : {}),
+ })),
+ );
+}
+
+function toPublicOutputDefinition(outputDefinition: unknown) {
+ const parsed =
+ PersistedEvalOutputDefinitionSchema.safeParse(outputDefinition);
+ if (!parsed.success) {
+ logger.error("Failed to parse public evaluator output definition", {
+ issues: parsed.error.issues,
+ });
+ throw new InternalServerError("Evaluator output definition is corrupted");
+ }
+
+ const resolved = resolvePersistedEvalOutputDefinition(parsed.data);
+ const descriptions = {
+ ...(resolved.reasoningDescription
+ ? { scoreReasoningInstructions: resolved.reasoningDescription }
+ : {}),
+ ...(resolved.scoreDescription
+ ? { scoreValueInstructions: resolved.scoreDescription }
+ : {}),
+ };
+ if (resolved.dataType === "NUMERIC") {
+ return {
+ dataType: resolved.dataType,
+ ...descriptions,
+ ...(resolved.minValue === undefined
+ ? {}
+ : { minValue: resolved.minValue }),
+ ...(resolved.maxValue === undefined
+ ? {}
+ : { maxValue: resolved.maxValue }),
+ };
+ }
+ if (resolved.dataType === "BOOLEAN") {
+ return { dataType: resolved.dataType, ...descriptions };
+ }
+ return {
+ dataType: resolved.dataType,
+ ...descriptions,
+ categories: resolved.categories,
+ shouldAllowMultipleMatches: resolved.shouldAllowMultipleMatches,
+ };
+}
+
+function toCreator(
+ creator: {
+ id: string;
+ name: string | null;
+ } | null,
+) {
+ return creator ? { id: creator.id, name: creator.name } : null;
+}
+
+function toInternalOutputDefinition(
+ outputDefinition: PublicEvaluatorOutputDefinitionType,
+) {
+ const descriptions = {
+ reasoning: {
+ description: outputDefinition.scoreReasoningInstructions ?? "",
+ },
+ scoreDescription: outputDefinition.scoreValueInstructions ?? "",
+ };
+ if (outputDefinition.dataType === "NUMERIC") {
+ return {
+ dataType: outputDefinition.dataType,
+ reasoning: descriptions.reasoning,
+ score: {
+ description: descriptions.scoreDescription,
+ ...(outputDefinition.minValue === undefined
+ ? {}
+ : { minValue: outputDefinition.minValue }),
+ ...(outputDefinition.maxValue === undefined
+ ? {}
+ : { maxValue: outputDefinition.maxValue }),
+ },
+ };
+ }
+ if (outputDefinition.dataType === "BOOLEAN") {
+ return {
+ dataType: outputDefinition.dataType,
+ reasoning: descriptions.reasoning,
+ score: { description: descriptions.scoreDescription },
+ };
+ }
+ return {
+ dataType: outputDefinition.dataType,
+ reasoning: descriptions.reasoning,
+ score: {
+ description: descriptions.scoreDescription,
+ categories: outputDefinition.categories,
+ shouldAllowMultipleMatches: outputDefinition.shouldAllowMultipleMatches,
+ },
+ };
+}
+
+export function toEvaluatorServiceDefinition(
+ definition: EvaluatorDefinitionType,
+) {
+ return EvaluatorDefinitionInputSchema.parse(
+ definition.type === PUBLIC_EVALUATOR_TYPE_CODE
+ ? {
+ type: EvalTemplateType.CODE,
+ sourceCode: definition.sourceCode,
+ sourceCodeLanguage: definition.sourceCodeLanguage,
+ }
+ : {
+ type: EvalTemplateType.LLM_AS_JUDGE,
+ prompt: definition.prompt[0]!.content,
+ modelConfig: definition.modelConfig ?? null,
+ variableMapping:
+ definition.variableMapping == null
+ ? null
+ : toStoredMappingList(definition.variableMapping),
+ outputDefinition: toInternalOutputDefinition(
+ definition.outputDefinition,
+ ),
+ },
+ );
+}
+
+type ServiceEvaluator = Awaited<ReturnType<EvaluatorService["get"]>>;
+type ServiceEvaluatorVersion =
+ | ServiceEvaluator["versions"][number]
+ | Awaited<ReturnType<EvaluatorService["listVersions"]>>["data"][number];
+
+export function toPublicEvaluatorVersion(
+ evaluatorType: EvalTemplateType,
+ version: ServiceEvaluatorVersion,
+) {
+ const common = {
+ id: version.id,
+ version: version.version,
+ createdAt: version.createdAt,
+ createdBy: toCreator(version.createdByUser),
+ };
+
+ if (evaluatorType === EvalTemplateType.CODE) {
+ if (!version.sourceCode || !version.sourceCodeLanguage) {
+ throw new InternalServerError("Code evaluator definition is corrupted");
+ }
+ return EvaluatorVersion.parse({
+ ...common,
+ type: PUBLIC_EVALUATOR_TYPE_CODE,
+ sourceCode: version.sourceCode,
+ sourceCodeLanguage: version.sourceCodeLanguage,
+ });
+ }
+
+ if (!version.prompt) {
+ throw new InternalServerError("Evaluator prompt is corrupted");
+ }
+ const prompt = [{ role: "user" as const, content: version.prompt }];
+ return EvaluatorVersion.parse({
+ ...common,
+ type: PUBLIC_EVALUATOR_TYPE_LLM_AS_JUDGE,
+ prompt,
+ variables:
+ version.vars.length > 0
+ ? version.vars
+ : extractVariables(prompt.map(({ content }) => content).join("\n")),
+ variableMapping:
+ version.variableMapping === null
+ ? null
+ : toApiReadMappings(version.variableMapping),
+ modelConfig:
+ version.provider && version.model
+ ? {
+ provider: version.provider,
+ model: version.model,
+ }
+ : null,
+ outputDefinition: toPublicOutputDefinition(version.outputDefinition),
+ });
+}
+
+export function toPublicEvaluator(evaluator: ServiceEvaluator) {
+ const latestVersion = evaluator.versions[0];
+ if (!latestVersion) {
+ throw new InternalServerError("Evaluator version is missing");
+ }
+ const common = {
+ id: evaluator.id,
+ name: evaluator.name,
+ description: evaluator.description,
+ createdBy: toCreator(evaluator.createdByUser),
+ status: evaluator.blockedAt ? ("paused" as const) : ("active" as const),
+ pausedAt: evaluator.blockedAt,
+ pausedReason: evaluator.blockReason,
+ pausedMessage: evaluator.blockMessage,
+ evaluationRuleAssignments: evaluator.assignments.map(
+ ({ evaluationRule, variableMapping }) => ({
+ evaluationRuleId: evaluationRule.id,
+ ...(variableMapping === null
+ ? {}
+ : {
+ variableMappingOverride: isLegacyEvalTarget(
+ evaluationRule.targetObject,
+ )
+ ? toApiLegacyMappings(variableMapping)
+ : toApiReadMappings(variableMapping),
+ }),
+ }),
+ ),
+ createdAt: evaluator.createdAt,
+ updatedAt: evaluator.updatedAt,
+ };
+ const {
+ id: versionId,
+ createdAt: versionCreatedAt,
+ createdBy: versionCreatedBy,
+ ...version
+ } = toPublicEvaluatorVersion(evaluator.type, latestVersion);
+ return Evaluator.parse({
+ ...common,
+ ...version,
+ versionId,
+ versionCreatedAt,
+ versionCreatedBy,
+ });
+}
+
+export function toInternalFilters(filters: FilterState) {
+ return filters.map((filter) =>
+ filter.column === "datasetId"
+ ? { ...filter, column: "experimentDatasetId" }
+ : filter,
+ ) as FilterState;
+}
+
+function toPublicFilters(filters: unknown) {
+ return PublicEvaluationRuleReadFilter.array()
+ .parse(filters)
+ .map((filter) =>
+ filter.column === "experimentDatasetId"
+ ? { ...filter, column: "datasetId" }
+ : filter,
+ );
+}
+
+export function toInternalAssignments(
+ assignments: CreateEvaluationRuleBodyType["evaluatorAssignments"],
+) {
+ return assignments.map((assignment) => ({
+ evaluatorId: assignment.evaluatorId,
+ variableMapping:
+ assignment.variableMapping === null
+ ? null
+ : (toStoredMappingList(
+ assignment.variableMapping,
+ ) as ObservationVariableMapping[]),
+ }));
+}
+
+type ServiceRule = Awaited<ReturnType<RuleService["get"]>>;
+
+export function toPublicRule(rule: ServiceRule) {
+ return EvaluationRule.parse({
+ id: rule.id,
+ name: rule.name,
+ createdBy: toCreator(rule.createdByUser),
+ enabled: rule.enabled,
+ sampling: rule.sampling,
+ filter: toPublicFilters(rule.filter),
+ evaluatorAssignments: rule.assignments.map((assignment) => ({
+ evaluatorId: assignment.evaluator.id,
+ variableMapping:
+ assignment.variableMapping === null
+ ? null
+ : isLegacyEvalTarget(rule.targetObject)
+ ? toApiLegacyMappings(assignment.variableMapping)
+ : toApiReadMappings(assignment.variableMapping),
+ })),
+ createdAt: rule.createdAt,
+ updatedAt: rule.updatedAt,
+ });
+}
diff --git a/web/src/features/public-api/server/evaluation/evaluationRuleApiHandler.ts b/web/src/features/public-api/server/evaluation/evaluationRuleApiHandler.ts
new file mode 100644
index 0000000000..f8a087d6e1
--- /dev/null
+++ b/web/src/features/public-api/server/evaluation/evaluationRuleApiHandler.ts
@@ -0,0 +1,53 @@
+import {
+ createStablePublicApiRoute,
+ withStablePublicApiMiddlewares,
+} from "@/src/features/public-api/server/stablePublicApiRoute";
+import {
+ deleteEvaluationRuleForPublicApi,
+ getEvaluationRuleForPublicApi,
+ updateEvaluationRuleForPublicApi,
+} from "./evaluationRuleApiService";
+import {
+ DeleteEvaluationRuleResponse,
+ EvaluationRule,
+ EvaluationRuleIdQuery,
+ UpdateEvaluationRuleBody,
+} from "@/src/features/public-api/types/evaluation/evaluationRules";
+
+export const evaluationRuleApiHandler = withStablePublicApiMiddlewares({
+ GET: createStablePublicApiRoute({
+ name: "Get evaluation rule",
+ querySchema: EvaluationRuleIdQuery,
+ responseSchema: EvaluationRule,
+ fn: ({ query, auth }) =>
+ getEvaluationRuleForPublicApi({
+ projectId: auth.scope.projectId,
+ evaluationRuleId: query.evaluationRuleId,
+ auditScope: auth.scope,
+ }),
+ }),
+ PATCH: createStablePublicApiRoute({
+ name: "Update evaluation rule",
+ querySchema: EvaluationRuleIdQuery,
+ bodySchema: UpdateEvaluationRuleBody,
+ responseSchema: EvaluationRule,
+ fn: ({ query, body, auth }) =>
+ updateEvaluationRuleForPublicApi({
+ projectId: auth.scope.projectId,
+ evaluationRuleId: query.evaluationRuleId,
+ input: body,
+ auditScope: auth.scope,
+ }),
+ }),
+ DELETE: createStablePublicApiRoute({
+ name: "Delete evaluation rule",
+ querySchema: EvaluationRuleIdQuery,
+ responseSchema: DeleteEvaluationRuleResponse,
+ fn: ({ query, auth }) =>
+ deleteEvaluationRuleForPublicApi({
+ projectId: auth.scope.projectId,
+ evaluationRuleId: query.evaluationRuleId,
+ auditScope: auth.scope,
+ }),
+ }),
+});
diff --git a/web/src/features/public-api/server/evaluation/evaluationRuleApiService.ts b/web/src/features/public-api/server/evaluation/evaluationRuleApiService.ts
new file mode 100644
index 0000000000..8ca5ae0043
--- /dev/null
+++ b/web/src/features/public-api/server/evaluation/evaluationRuleApiService.ts
@@ -0,0 +1,135 @@
+import { EvalTargetObject } from "@langfuse/shared";
+import { prisma } from "@langfuse/shared/src/db";
+import type { ApiAccessScope } from "@langfuse/shared/src/server";
+import { auditLog } from "@/src/features/audit-logs/server";
+import {
+ JOB_CONFIGURATION_AUDIT_LOG_RESOURCE_TYPE,
+ RuleService,
+} from "@/src/features/evals/server";
+import {
+ type CreateEvaluationRuleBodyType,
+ type UpdateEvaluationRuleBodyType,
+} from "@/src/features/public-api/types/evaluation/evaluationRules";
+import {
+ encodeResourceCursor,
+ type ResourceCursorType,
+} from "@/src/features/public-api/types/evaluation/evaluators";
+import {
+ toInternalAssignments,
+ toInternalFilters,
+ toPublicRule,
+} from "./evaluationAdapters";
+
+function ruleService(auditScope: ApiAccessScope) {
+ return new RuleService(prisma, ({ action, ruleId, projectId }) =>
+ auditLog({
+ action,
+ resourceType: JOB_CONFIGURATION_AUDIT_LOG_RESOURCE_TYPE,
+ resourceId: ruleId,
+ projectId,
+ orgId: auditScope.orgId,
+ apiKeyId: auditScope.apiKeyId,
+ }),
+ );
+}
+
+export async function listEvaluationRulesForPublicApi(params: {
+ projectId: string;
+ limit: number;
+ cursor?: ResourceCursorType;
+ auditScope: ApiAccessScope;
+}) {
+ const result = await ruleService(params.auditScope).listCursor({
+ projectId: params.projectId,
+ limit: params.limit,
+ cursor: params.cursor
+ ? {
+ createdAt: new Date(params.cursor.lastCreatedAt),
+ id: params.cursor.lastId,
+ }
+ : undefined,
+ });
+ return {
+ data: result.rules.map(toPublicRule),
+ meta: result.nextCursor
+ ? {
+ cursor: encodeResourceCursor({
+ v: 1,
+ lastCreatedAt: result.nextCursor.createdAt.toISOString(),
+ lastId: result.nextCursor.id,
+ }),
+ }
+ : {},
+ };
+}
+
+export async function getEvaluationRuleForPublicApi(params: {
+ projectId: string;
+ evaluationRuleId: string;
+ auditScope: ApiAccessScope;
+}) {
+ return toPublicRule(
+ await ruleService(params.auditScope).get(
+ params.projectId,
+ params.evaluationRuleId,
+ ),
+ );
+}
+
+export async function createEvaluationRuleForPublicApi(params: {
+ projectId: string;
+ input: CreateEvaluationRuleBodyType;
+ auditScope: ApiAccessScope;
+}) {
+ return toPublicRule(
+ await ruleService(params.auditScope).create(
+ {
+ projectId: params.projectId,
+ name: params.input.name,
+ targetObject: EvalTargetObject.EVENT,
+ enabled: params.input.enabled,
+ sampling: params.input.sampling,
+ filter: toInternalFilters(params.input.filter),
+ evaluatorAssignments: toInternalAssignments(
+ params.input.evaluatorAssignments,
+ ),
+ },
+ null,
+ ),
+ );
+}
+
+export async function updateEvaluationRuleForPublicApi(params: {
+ projectId: string;
+ evaluationRuleId: string;
+ input: UpdateEvaluationRuleBodyType;
+ auditScope: ApiAccessScope;
+}) {
+ return toPublicRule(
+ await ruleService(params.auditScope).update({
+ projectId: params.projectId,
+ ruleId: params.evaluationRuleId,
+ name: params.input.name,
+ enabled: params.input.enabled,
+ sampling: params.input.sampling,
+ filter: params.input.filter
+ ? toInternalFilters(params.input.filter)
+ : undefined,
+ evaluatorMappings: params.input.evaluatorAssignments
+ ? toInternalAssignments(params.input.evaluatorAssignments)
+ : undefined,
+ }),
+ );
+}
+
+export async function deleteEvaluationRuleForPublicApi(params: {
+ projectId: string;
+ evaluationRuleId: string;
+ auditScope: ApiAccessScope;
+}) {
+ await ruleService(params.auditScope).delete(
+ params.projectId,
+ params.evaluationRuleId,
+ );
+ return { id: params.evaluationRuleId };
+}
diff --git a/web/src/features/public-api/server/evaluation/evaluationRulesApiHandler.ts b/web/src/features/public-api/server/evaluation/evaluationRulesApiHandler.ts
new file mode 100644
index 0000000000..ec8910e300
--- /dev/null
+++ b/web/src/features/public-api/server/evaluation/evaluationRulesApiHandler.ts
@@ -0,0 +1,41 @@
+import {
+ createStablePublicApiRoute,
+ withStablePublicApiMiddlewares,
+} from "@/src/features/public-api/server/stablePublicApiRoute";
+import {
+ createEvaluationRuleForPublicApi,
+ listEvaluationRulesForPublicApi,
+} from "./evaluationRuleApiService";
+import {
+ CreateEvaluationRuleBody,
+ EvaluationRule,
+ ListEvaluationRulesQuery,
+ ListEvaluationRulesResponse,
+} from "@/src/features/public-api/types/evaluation/evaluationRules";
+
+export const evaluationRulesApiHandler = withStablePublicApiMiddlewares({
+ GET: createStablePublicApiRoute({
+ name: "List evaluation rules",
+ querySchema: ListEvaluationRulesQuery,
+ responseSchema: ListEvaluationRulesResponse,
+ fn: ({ query, auth }) =>
+ listEvaluationRulesForPublicApi({
+ projectId: auth.scope.projectId,
+ limit: query.limit,
+ cursor: query.cursor,
+ auditScope: auth.scope,
+ }),
+ }),
+ POST: createStablePublicApiRoute({
+ name: "Create evaluation rule",
+ bodySchema: CreateEvaluationRuleBody,
+ responseSchema: EvaluationRule,
+ successStatusCode: 201,
+ fn: ({ body, auth }) =>
+ createEvaluationRuleForPublicApi({
+ projectId: auth.scope.projectId,
+ input: body,
+ auditScope: auth.scope,
+ }),
+ }),
+});
diff --git a/web/src/features/public-api/server/evaluation/evaluatorApiHandler.ts b/web/src/features/public-api/server/evaluation/evaluatorApiHandler.ts
new file mode 100644
index 0000000000..310b204b6e
--- /dev/null
+++ b/web/src/features/public-api/server/evaluation/evaluatorApiHandler.ts
@@ -0,0 +1,53 @@
+import {
+ createStablePublicApiRoute,
+ withStablePublicApiMiddlewares,
+} from "@/src/features/public-api/server/stablePublicApiRoute";
+import {
+ deleteEvaluatorForPublicApi,
+ getEvaluatorForPublicApi,
+ updateEvaluatorForPublicApi,
+} from "./evaluatorApiService";
+import {
+ DeleteEvaluatorResponse,
+ Evaluator,
+ EvaluatorIdQuery,
+ UpdateEvaluatorBody,
+} from "@/src/features/public-api/types/evaluation/evaluators";
+
+export const evaluatorApiHandler = withStablePublicApiMiddlewares({
+ GET: createStablePublicApiRoute({
+ name: "Get evaluator",
+ querySchema: EvaluatorIdQuery,
+ responseSchema: Evaluator,
+ fn: ({ query, auth }) =>
+ getEvaluatorForPublicApi({
+ projectId: auth.scope.projectId,
+ evaluatorId: query.evaluatorId,
+ auditScope: auth.scope,
+ }),
+ }),
+ PATCH: createStablePublicApiRoute({
+ name: "Update evaluator",
+ querySchema: EvaluatorIdQuery,
+ bodySchema: UpdateEvaluatorBody,
+ responseSchema: Evaluator,
+ fn: ({ query, body, auth }) =>
+ updateEvaluatorForPublicApi({
+ projectId: auth.scope.projectId,
+ evaluatorId: query.evaluatorId,
+ input: body,
+ auditScope: auth.scope,
+ }),
+ }),
+ DELETE: createStablePublicApiRoute({
+ name: "Delete evaluator",
+ querySchema: EvaluatorIdQuery,
+ responseSchema: DeleteEvaluatorResponse,
+ fn: ({ query, auth }) =>
+ deleteEvaluatorForPublicApi({
+ projectId: auth.scope.projectId,
+ evaluatorId: query.evaluatorId,
+ auditScope: auth.scope,
+ }),
+ }),
+});
diff --git a/web/src/features/public-api/server/evaluation/evaluatorApiService.ts b/web/src/features/public-api/server/evaluation/evaluatorApiService.ts
new file mode 100644
index 0000000000..a319e4836f
--- /dev/null
+++ b/web/src/features/public-api/server/evaluation/evaluatorApiService.ts
@@ -0,0 +1,170 @@
+import { EvalTemplateType, ZodModelConfig } from "@langfuse/shared";
+import { prisma } from "@langfuse/shared/src/db";
+import type { ApiAccessScope } from "@langfuse/shared/src/server";
+import { auditLog } from "@/src/features/audit-logs/server";
+import {
+ EVAL_TEMPLATE_AUDIT_LOG_RESOURCE_TYPE,
+ EvaluatorService,
+} from "@/src/features/evals/server";
+import {
+ encodeResourceCursor,
+ type CreateEvaluatorBodyType,
+ type ResourceCursorType,
+ type UpdateEvaluatorBodyType,
+} from "@/src/features/public-api/types/evaluation/evaluators";
+import {
+ toEvaluatorServiceDefinition,
+ toPublicEvaluator,
+ toPublicEvaluatorVersion,
+} from "./evaluationAdapters";
+
+function evaluatorService(auditScope: ApiAccessScope) {
+ return new EvaluatorService(prisma, ({ action, evaluatorId, projectId }) =>
+ auditLog({
+ action,
+ resourceType: EVAL_TEMPLATE_AUDIT_LOG_RESOURCE_TYPE,
+ resourceId: evaluatorId,
+ projectId,
+ orgId: auditScope.orgId,
+ apiKeyId: auditScope.apiKeyId,
+ }),
+ );
+}
+
+export async function listEvaluatorsForPublicApi(params: {
+ projectId: string;
+ limit: number;
+ cursor?: ResourceCursorType;
+ auditScope: ApiAccessScope;
+}) {
+ const result = await evaluatorService(params.auditScope).listCursor({
+ projectId: params.projectId,
+ limit: params.limit,
+ cursor: params.cursor
+ ? {
+ createdAt: new Date(params.cursor.lastCreatedAt),
+ id: params.cursor.lastId,
+ }
+ : undefined,
+ });
+ return {
+ data: result.evaluators.map(toPublicEvaluator),
+ meta: result.nextCursor
+ ? {
+ cursor: encodeResourceCursor({
+ v: 1,
+ lastCreatedAt: result.nextCursor.createdAt.toISOString(),
+ lastId: result.nextCursor.id,
+ }),
+ }
+ : {},
+ };
+}
+
+export async function getEvaluatorForPublicApi(params: {
+ projectId: string;
+ evaluatorId: string;
+ auditScope: ApiAccessScope;
+}) {
+ return toPublicEvaluator(
+ await evaluatorService(params.auditScope).get(
+ params.projectId,
+ params.evaluatorId,
+ ),
+ );
+}
+
+export async function createEvaluatorForPublicApi(params: {
+ projectId: string;
+ input: CreateEvaluatorBodyType;
+ auditScope: ApiAccessScope;
+}) {
+ return toPublicEvaluator(
+ await evaluatorService(params.auditScope).create(
+ {
+ projectId: params.projectId,
+ name: params.input.name,
+ description: params.input.description ?? null,
+ definition: toEvaluatorServiceDefinition(params.input),
+ },
+ null,
+ ),
+ );
+}
+
+export async function updateEvaluatorForPublicApi(params: {
+ projectId: string;
+ evaluatorId: string;
+ input: UpdateEvaluatorBodyType;
+ auditScope: ApiAccessScope;
+}) {
+ const service = evaluatorService(params.auditScope);
+ let definition =
+ "type" in params.input
+ ? toEvaluatorServiceDefinition(params.input)
+ : undefined;
+
+ if (
+ definition?.type === EvalTemplateType.LLM_AS_JUDGE &&
+ "type" in params.input &&
+ params.input.type === "llm_as_judge" &&
+ params.input.modelConfig === undefined
+ ) {
+ const current = await service.get(params.projectId, params.evaluatorId);
+ const latestVersion = current.versions[0];
+ if (current.type === EvalTemplateType.LLM_AS_JUDGE && latestVersion) {
+ definition = {
+ ...definition,
+ provider: latestVersion.provider,
+ model: latestVersion.model,
+ modelParams: ZodModelConfig.nullable().parse(latestVersion.modelParams),
+ };
+ }
+ }
+
+ return toPublicEvaluator(
+ await service.patch(
+ {
+ projectId: params.projectId,
+ evaluatorId: params.evaluatorId,
+ name: params.input.name,
+ description: params.input.description,
+ definition,
+ },
+ null,
+ ),
+ );
+}
+
+export async function deleteEvaluatorForPublicApi(params: {
+ projectId: string;
+ evaluatorId: string;
+ auditScope: ApiAccessScope;
+}) {
+ await evaluatorService(params.auditScope).delete(
+ params.projectId,
+ params.evaluatorId,
+ );
+ return { id: params.evaluatorId };
+}
+
+export async function listEvaluatorVersionsForPublicApi(params: {
+ projectId: string;
+ evaluatorId: string;
+ limit: number;
+ cursor?: { v: 1; version: number };
+ auditScope: ApiAccessScope;
+}) {
+ const result = await evaluatorService(params.auditScope).listVersions({
+ projectId: params.projectId,
+ evaluatorId: params.evaluatorId,
+ limit: params.limit,
+ cursor: params.cursor,
+ });
+ return {
+ data: result.data.map((version) =>
+ toPublicEvaluatorVersion(version.evaluator.type, version),
+ ),
+ meta: result.nextCursor ? { cursor: result.nextCursor } : {},
+ };
+}
diff --git a/web/src/features/public-api/server/evaluation/evaluatorVersionsApiHandler.ts b/web/src/features/public-api/server/evaluation/evaluatorVersionsApiHandler.ts
new file mode 100644
index 0000000000..77650d8acc
--- /dev/null
+++ b/web/src/features/public-api/server/evaluation/evaluatorVersionsApiHandler.ts
@@ -0,0 +1,25 @@
+import {
+ createStablePublicApiRoute,
+ withStablePublicApiMiddlewares,
+} from "@/src/features/public-api/server/stablePublicApiRoute";
+import { listEvaluatorVersionsForPublicApi } from "./evaluatorApiService";
+import {
+ ListEvaluatorVersionsQuery,
+ ListEvaluatorVersionsResponse,
+} from "@/src/features/public-api/types/evaluation/evaluators";
+
+export const evaluatorVersionsApiHandler = withStablePublicApiMiddlewares({
+ GET: createStablePublicApiRoute({
+ name: "List evaluator versions",
+ querySchema: ListEvaluatorVersionsQuery,
+ responseSchema: ListEvaluatorVersionsResponse,
+ fn: ({ query, auth }) =>
+ listEvaluatorVersionsForPublicApi({
+ projectId: auth.scope.projectId,
+ evaluatorId: query.evaluatorId,
+ limit: query.limit,
+ cursor: query.cursor,
+ auditScope: auth.scope,
+ }),
+ }),
+});
diff --git a/web/src/features/public-api/server/evaluation/evaluatorsApiHandler.ts b/web/src/features/public-api/server/evaluation/evaluatorsApiHandler.ts
new file mode 100644
index 0000000000..5a6cceac47
--- /dev/null
+++ b/web/src/features/public-api/server/evaluation/evaluatorsApiHandler.ts
@@ -0,0 +1,41 @@
+import {
+ createStablePublicApiRoute,
+ withStablePublicApiMiddlewares,
+} from "@/src/features/public-api/server/stablePublicApiRoute";
+import {
+ createEvaluatorForPublicApi,
+ listEvaluatorsForPublicApi,
+} from "./evaluatorApiService";
+import {
+ CreateEvaluatorBody,
+ Evaluator,
+ ListEvaluatorsQuery,
+ ListEvaluatorsResponse,
+} from "@/src/features/public-api/types/evaluation/evaluators";
+
+export const evaluatorsApiHandler = withStablePublicApiMiddlewares({
+ GET: createStablePublicApiRoute({
+ name: "List evaluators",
+ querySchema: ListEvaluatorsQuery,
+ responseSchema: ListEvaluatorsResponse,
+ fn: ({ query, auth }) =>
+ listEvaluatorsForPublicApi({
+ projectId: auth.scope.projectId,
+ limit: query.limit,
+ cursor: query.cursor,
+ auditScope: auth.scope,
+ }),
+ }),
+ POST: createStablePublicApiRoute({
+ name: "Create evaluator",
+ bodySchema: CreateEvaluatorBody,
+ responseSchema: Evaluator,
+ successStatusCode: 201,
+ fn: ({ body, auth }) =>
+ createEvaluatorForPublicApi({
+ projectId: auth.scope.projectId,
+ input: body,
+ auditScope: auth.scope,
+ }),
+ }),
+});
diff --git a/web/src/features/public-api/server/index.ts b/web/src/features/public-api/server/index.ts
new file mode 100644
index 0000000000..1e1b70fb86
--- /dev/null
+++ b/web/src/features/public-api/server/index.ts
@@ -0,0 +1,10 @@
+export { evaluationRulesApiHandler } from "./evaluation/evaluationRulesApiHandler";
+export { evaluationRuleApiHandler } from "./evaluation/evaluationRuleApiHandler";
+export { evaluatorsApiHandler } from "./evaluation/evaluatorsApiHandler";
+export { evaluatorApiHandler } from "./evaluation/evaluatorApiHandler";
+export { evaluatorVersionsApiHandler } from "./evaluation/evaluatorVersionsApiHandler";
+export {
+ createStructuredPublicApiRateLimitError,
+ structuredPublicApiErrorContract,
+ toStructuredPublicApiError,
+} from "./structuredPublicApiErrorContract";
diff --git a/web/src/features/public-api/server/stablePublicApiRoute.ts b/web/src/features/public-api/server/stablePublicApiRoute.ts
new file mode 100644
index 0000000000..82b1771af7
--- /dev/null
+++ b/web/src/features/public-api/server/stablePublicApiRoute.ts
@@ -0,0 +1,46 @@
+import { type NextApiRequest, type NextApiResponse } from "next";
+import { type ZodType } from "zod";
+import {
+ createAuthedProjectAPIRoute,
+ type AuthedProjectAPIRouteConfig,
+} from "@/src/features/public-api/server/createAuthedProjectAPIRoute";
+import {
+ withMiddlewares,
+ type HttpMethod,
+} from "@/src/features/public-api/server/withMiddlewares";
+import { structuredPublicApiErrorContract } from "./structuredPublicApiErrorContract";
+
+type StablePublicApiRouteConfig<
+ TQuery extends ZodType<any>,
+ TBody extends ZodType<any>,
+ TResponse extends ZodType<any>,
+> = Omit<
+ AuthedProjectAPIRouteConfig<TQuery, TBody, TResponse>,
+ "errorContract"
+>;
+
+type StablePublicApiHandlers = {
+ [Method in HttpMethod]?: (
+ req: NextApiRequest,
+ res: NextApiResponse,
+ ) => Promise<void>;
+};
+
+export const createStablePublicApiRoute = <
+ TQuery extends ZodType<any>,
+ TBody extends ZodType<any>,
+ TResponse extends ZodType<any>,
+>(
+ routeConfig: StablePublicApiRouteConfig<TQuery, TBody, TResponse>,
+) =>
+ createAuthedProjectAPIRoute({
+ ...routeConfig,
+ errorContract: structuredPublicApiErrorContract,
+ });
+
+export const withStablePublicApiMiddlewares = (
+ handlers: StablePublicApiHandlers,
+) =>
+ withMiddlewares(handlers, {
+ errorContract: structuredPublicApiErrorContract,
+ });
diff --git a/web/src/features/public-api/server/unstable-public-api-error-contract.ts b/web/src/features/public-api/server/structuredPublicApiErrorContract.ts
similarity index 65%
rename from web/src/features/public-api/server/unstable-public-api-error-contract.ts
rename to web/src/features/public-api/server/structuredPublicApiErrorContract.ts
index 6454d5bb63..bd9e4b85d3 100644
--- a/web/src/features/public-api/server/unstable-public-api-error-contract.ts
+++ b/web/src/features/public-api/server/structuredPublicApiErrorContract.ts
@@ -12,41 +12,31 @@ import {
type RateLimitResult,
} from "@langfuse/shared";
import { ClickHouseResourceError } from "@langfuse/shared/src/server";
+import {
+ createStructuredPublicApiError,
+ StructuredPublicApiError,
+} from "@/src/features/public-api/types/structuredPublicApiError";
import type {
- UnstablePublicApiErrorCodeType,
- UnstablePublicApiErrorDetailsType,
-} from "@/src/features/public-api/shared/unstable-public-api-error-schema";
+ StructuredPublicApiErrorCodeType,
+ StructuredPublicApiErrorDetailsType,
+} from "@/src/features/public-api/types/structuredPublicApiErrorSchema";
import {
getRateLimitUpgradeMessage,
type RateLimitUpgradePath,
} from "@/src/features/public-api/server/rateLimitUpgradePaths";
-export const unstablePublicEvalsErrorContract = "unstable-public-evals";
-export type PublicApiErrorContract = typeof unstablePublicEvalsErrorContract;
+export const structuredPublicApiErrorContract = "structured";
+export type PublicApiErrorContract = typeof structuredPublicApiErrorContract;
-type UnstablePublicApiErrorBody = {
+type StructuredPublicApiErrorBody = {
message: string;
- code: UnstablePublicApiErrorCodeType;
- details?: UnstablePublicApiErrorDetailsType;
+ code: StructuredPublicApiErrorCodeType;
+ details?: StructuredPublicApiErrorDetailsType;
};
-export class UnstablePublicApiError extends BaseError {
- public readonly code: UnstablePublicApiErrorCodeType;
- public readonly details?: UnstablePublicApiErrorDetailsType;
+export { StructuredPublicApiError };
- constructor(params: {
- httpCode: number;
- code: UnstablePublicApiErrorCodeType;
- message: string;
- details?: UnstablePublicApiErrorDetailsType;
- }) {
- super("UnstablePublicApiError", params.httpCode, params.message, true);
- this.code = params.code;
- this.details = params.details;
- }
-}
-
-function toBody(error: UnstablePublicApiError): UnstablePublicApiErrorBody {
+function toBody(error: StructuredPublicApiError): StructuredPublicApiErrorBody {
return {
message: error.message,
code: error.code,
@@ -64,53 +54,44 @@ function toSerializableIssues(issues: ZodError["issues"]) {
}));
}
-export function sendUnstablePublicApiErrorResponse(
+export function sendStructuredPublicApiErrorResponse(
res: NextApiResponse,
- error: UnstablePublicApiError,
+ error: StructuredPublicApiError,
) {
return res.status(error.httpCode).json(toBody(error));
}
-export function createUnstablePublicApiError(params: {
- httpCode: number;
- code: UnstablePublicApiErrorCodeType;
- message: string;
- details?: UnstablePublicApiErrorDetailsType;
-}) {
- return new UnstablePublicApiError(params);
-}
-
-export function createUnstablePublicApiAuthError(params: {
+export function createStructuredPublicApiAuthError(params: {
statusCode: number;
message: string;
}) {
switch (params.statusCode) {
case 400:
- return createUnstablePublicApiError({
+ return createStructuredPublicApiError({
httpCode: 400,
code: "invalid_request",
message: params.message,
});
case 401:
- return createUnstablePublicApiError({
+ return createStructuredPublicApiError({
httpCode: 401,
code: "authentication_failed",
message: params.message,
});
case 403:
- return createUnstablePublicApiError({
+ return createStructuredPublicApiError({
httpCode: 403,
code: "access_denied",
message: params.message,
});
case 404:
- return createUnstablePublicApiError({
+ return createStructuredPublicApiError({
httpCode: 404,
code: "resource_not_found",
message: params.message,
});
default:
- return createUnstablePublicApiError({
+ return createStructuredPublicApiError({
httpCode: params.statusCode,
code: params.statusCode >= 500 ? "internal_error" : "invalid_request",
message: params.message,
@@ -118,14 +99,14 @@ export function createUnstablePublicApiAuthError(params: {
}
}
-export function createUnstablePublicApiRateLimitError(
+export function createStructuredPublicApiRateLimitError(
rateLimitRes: RateLimitResult,
options?: {
errorContract?: PublicApiErrorContract;
upgradePath?: RateLimitUpgradePath;
},
) {
- return createUnstablePublicApiError({
+ return createStructuredPublicApiError({
httpCode: 429,
code: "rate_limited",
message: options?.upgradePath
@@ -140,11 +121,11 @@ export function createUnstablePublicApiRateLimitError(
});
}
-export function createUnstablePublicApiRequestValidationError(params: {
+export function createStructuredPublicApiRequestValidationError(params: {
error: ZodError;
requestPart: "query" | "body";
}) {
- return createUnstablePublicApiError({
+ return createStructuredPublicApiError({
httpCode: 400,
code: params.requestPart === "query" ? "invalid_query" : "invalid_body",
message:
@@ -157,10 +138,10 @@ export function createUnstablePublicApiRequestValidationError(params: {
});
}
-export function toUnstablePublicApiError(
+export function toStructuredPublicApiError(
error: unknown,
-): UnstablePublicApiError {
- if (error instanceof UnstablePublicApiError) {
+): StructuredPublicApiError {
+ if (error instanceof StructuredPublicApiError) {
return error;
}
@@ -169,7 +150,7 @@ export function toUnstablePublicApiError(
error.constructor.name === "ZodError" &&
"issues" in error
) {
- return createUnstablePublicApiError({
+ return createStructuredPublicApiError({
httpCode: 400,
code: "invalid_request",
message: "Invalid request data",
@@ -180,7 +161,7 @@ export function toUnstablePublicApiError(
}
if (error instanceof LangfuseNotFoundError) {
- return createUnstablePublicApiError({
+ return createStructuredPublicApiError({
httpCode: 404,
code: "resource_not_found",
message: error.message,
@@ -188,7 +169,7 @@ export function toUnstablePublicApiError(
}
if (error instanceof UnauthorizedError) {
- return createUnstablePublicApiError({
+ return createStructuredPublicApiError({
httpCode: 403,
code: "access_denied",
message: error.message,
@@ -196,7 +177,7 @@ export function toUnstablePublicApiError(
}
if (error instanceof ForbiddenError) {
- return createUnstablePublicApiError({
+ return createStructuredPublicApiError({
httpCode: 403,
code: "access_denied",
message: error.message,
@@ -204,7 +185,7 @@ export function toUnstablePublicApiError(
}
if (error instanceof MethodNotAllowedError) {
- return createUnstablePublicApiError({
+ return createStructuredPublicApiError({
httpCode: 405,
code: "method_not_allowed",
message: error.message,
@@ -212,7 +193,7 @@ export function toUnstablePublicApiError(
}
if (error instanceof LangfuseConflictError) {
- return createUnstablePublicApiError({
+ return createStructuredPublicApiError({
httpCode: 409,
code: "conflict",
message: error.message,
@@ -220,7 +201,7 @@ export function toUnstablePublicApiError(
}
if (error instanceof InvalidRequestError) {
- return createUnstablePublicApiError({
+ return createStructuredPublicApiError({
httpCode: 400,
code: "invalid_request",
message: error.message,
@@ -228,9 +209,9 @@ export function toUnstablePublicApiError(
}
if (error instanceof ClickHouseResourceError) {
- return createUnstablePublicApiError({
+ return createStructuredPublicApiError({
httpCode: 422,
- code: "unprocessable_content",
+ code: "invalid_request",
message: [
ClickHouseResourceError.ERROR_ADVICE_MESSAGE,
"See https://langfuse.com/docs/api-and-data-platform/features/public-api for more details.",
@@ -239,7 +220,7 @@ export function toUnstablePublicApiError(
}
if (error instanceof InternalServerError) {
- return createUnstablePublicApiError({
+ return createStructuredPublicApiError({
httpCode: 500,
code: "internal_error",
message: error.message,
@@ -247,14 +228,14 @@ export function toUnstablePublicApiError(
}
if (error instanceof BaseError) {
- return createUnstablePublicApiError({
+ return createStructuredPublicApiError({
httpCode: error.httpCode,
code: error.httpCode >= 500 ? "internal_error" : "invalid_request",
message: error.message,
});
}
- return createUnstablePublicApiError({
+ return createStructuredPublicApiError({
httpCode: 500,
code: "internal_error",
message: "Internal Server Error",
diff --git a/web/src/features/public-api/server/unstable-public-api-route.ts b/web/src/features/public-api/server/unstable-public-api-route.ts
index 659c7fd324..a341afc27f 100644
--- a/web/src/features/public-api/server/unstable-public-api-route.ts
+++ b/web/src/features/public-api/server/unstable-public-api-route.ts
@@ -8,7 +8,7 @@ import {
withMiddlewares,
type HttpMethod,
} from "@/src/features/public-api/server/withMiddlewares";
-import { unstablePublicEvalsErrorContract } from "@/src/features/public-api/server/unstable-public-api-error-contract";
+import { structuredPublicApiErrorContract } from "./structuredPublicApiErrorContract";
type UnstablePublicApiRouteConfig<
TQuery extends ZodType<any>,
@@ -35,12 +35,12 @@ export const createUnstablePublicApiRoute = <
) =>
createAuthedProjectAPIRoute({
...routeConfig,
- errorContract: unstablePublicEvalsErrorContract,
+ errorContract: structuredPublicApiErrorContract,
});
export const withUnstablePublicApiMiddlewares = (
handlers: UnstablePublicApiHandlers,
) =>
withMiddlewares(handlers, {
- errorContract: unstablePublicEvalsErrorContract,
+ errorContract: structuredPublicApiErrorContract,
});
diff --git a/web/src/features/public-api/server/withMiddlewares.ts b/web/src/features/public-api/server/withMiddlewares.ts
index 9804e63af0..b7a90b0490 100644
--- a/web/src/features/public-api/server/withMiddlewares.ts
+++ b/web/src/features/public-api/server/withMiddlewares.ts
@@ -17,11 +17,11 @@ import {
} from "@langfuse/shared/src/server";
import * as opentelemetry from "@opentelemetry/api";
import {
- sendUnstablePublicApiErrorResponse,
- toUnstablePublicApiError,
- unstablePublicEvalsErrorContract,
+ sendStructuredPublicApiErrorResponse,
+ structuredPublicApiErrorContract,
+ toStructuredPublicApiError,
type PublicApiErrorContract,
-} from "@/src/features/public-api/server/unstable-public-api-error-contract";
+} from "./structuredPublicApiErrorContract";
import { clickHouseRouteForRequest } from "@/src/features/public-api/server/clickHouseRequestTags";
// eslint-disable-next-line @typescript-eslint/no-unused-vars -- Used via typeof
@@ -125,10 +125,10 @@ export function withMiddlewares(
tags: error.tags,
});
- if (options?.errorContract === unstablePublicEvalsErrorContract) {
- return sendUnstablePublicApiErrorResponse(
+ if (options?.errorContract === structuredPublicApiErrorContract) {
+ return sendStructuredPublicApiErrorResponse(
res,
- toUnstablePublicApiError(error),
+ toStructuredPublicApiError(error),
);
}
@@ -138,7 +138,7 @@ export function withMiddlewares(
});
}
- if (options?.errorContract === unstablePublicEvalsErrorContract) {
+ if (options?.errorContract === structuredPublicApiErrorContract) {
if (isBaseError(error)) {
logBaseError(error);
} else if (isZodError(error)) {
@@ -155,9 +155,9 @@ export function withMiddlewares(
traceException(error);
}
- return sendUnstablePublicApiErrorResponse(
+ return sendStructuredPublicApiErrorResponse(
res,
- toUnstablePublicApiError(error),
+ toStructuredPublicApiError(error),
);
}
diff --git a/web/src/features/public-api/types/evaluation/evaluationRules.ts b/web/src/features/public-api/types/evaluation/evaluationRules.ts
new file mode 100644
index 0000000000..38a81b2612
--- /dev/null
+++ b/web/src/features/public-api/types/evaluation/evaluationRules.ts
@@ -0,0 +1,111 @@
+import { publicApiPaginationLimitZod } from "@langfuse/shared";
+import { z } from "zod";
+import {
+ PromptVariableMapping,
+ PromptVariableMappingInput,
+ PublicEvaluationRuleReadFilter,
+ StableEvaluationRuleFilter,
+} from "./publicEvalsContract";
+import {
+ EncodedResourceCursor,
+ EncodedResourceCursorString,
+ PublicApiCreator,
+} from "./evaluators";
+
+const EvaluationRuleEvaluatorAssignmentInput = z
+ .object({
+ evaluatorId: z.string().min(1),
+ variableMapping: z
+ .array(PromptVariableMappingInput)
+ .nullable()
+ .default(null),
+ })
+ .strict();
+
+const EvaluatorAssignment = z
+ .object({
+ evaluatorId: z.string(),
+ variableMapping: z.array(PromptVariableMapping).nullable(),
+ })
+ .strict();
+
+export const EvaluationRule = z
+ .object({
+ id: z.string(),
+ name: z.string(),
+ createdBy: PublicApiCreator.nullable(),
+ enabled: z.boolean(),
+ sampling: z.number().min(0).max(1),
+ filter: z.array(PublicEvaluationRuleReadFilter),
+ evaluatorAssignments: z.array(EvaluatorAssignment),
+ createdAt: z.coerce.date(),
+ updatedAt: z.coerce.date(),
+ })
+ .strict();
+
+export const CreateEvaluationRuleBody = z
+ .object({
+ name: z.string().trim().min(1),
+ enabled: z.boolean(),
+ sampling: z.number().min(0).max(1).default(1),
+ filter: z.array(StableEvaluationRuleFilter).default([]),
+ evaluatorAssignments: z
+ .array(EvaluationRuleEvaluatorAssignmentInput)
+ .max(100),
+ })
+ .strict()
+ .superRefine((value, ctx) => {
+ if (value.enabled && value.evaluatorAssignments.length === 0) {
+ ctx.addIssue({
+ code: "custom",
+ message:
+ "An enabled evaluation rule requires at least one evaluator assignment",
+ path: ["evaluatorAssignments"],
+ });
+ }
+ });
+
+export const UpdateEvaluationRuleBody = z
+ .object({
+ name: z.string().trim().min(1).optional(),
+ enabled: z.boolean().optional(),
+ sampling: z.number().min(0).max(1).optional(),
+ filter: z.array(StableEvaluationRuleFilter).optional(),
+ evaluatorAssignments: z
+ .array(EvaluationRuleEvaluatorAssignmentInput)
+ .max(100)
+ .optional(),
+ })
+ .strict()
+ .refine((value) => Object.keys(value).length > 0, {
+ message: "Request body cannot be empty",
+ });
+
+export const ListEvaluationRulesQuery = z
+ .object({
+ limit: publicApiPaginationLimitZod,
+ cursor: EncodedResourceCursor.optional(),
+ })
+ .strict();
+
+export const ListEvaluationRulesResponse = z
+ .object({
+ data: z.array(EvaluationRule),
+ meta: z.object({ cursor: EncodedResourceCursorString.optional() }).strict(),
+ })
+ .strict();
+
+export const EvaluationRuleIdQuery = z
+ .object({ evaluationRuleId: z.string() })
+ .strict();
+
+export const DeleteEvaluationRuleResponse = z
+ .object({ id: z.string() })
+ .strict();
+
+export type CreateEvaluationRuleBodyType = z.infer<
+ typeof CreateEvaluationRuleBody
+>;
+export type UpdateEvaluationRuleBodyType = z.infer<
+ typeof UpdateEvaluationRuleBody
+>;
diff --git a/web/src/features/public-api/types/evaluation/evaluators.ts b/web/src/features/public-api/types/evaluation/evaluators.ts
new file mode 100644
index 0000000000..0067ed1d46
--- /dev/null
+++ b/web/src/features/public-api/types/evaluation/evaluators.ts
@@ -0,0 +1,241 @@
+import {
+ InvalidRequestError,
+ publicApiPaginationLimitZod,
+} from "@langfuse/shared";
+import { z } from "zod";
+import {
+ PUBLIC_EVALUATOR_TYPE_CODE,
+ PUBLIC_EVALUATOR_TYPE_LLM_AS_JUDGE,
+ PromptVariableMapping,
+ PublicCodeEvaluatorSourceCodeLanguage,
+ PromptVariableMappingInput,
+ PromptVariableMappingRead,
+ PublicEvaluatorOutputDefinition,
+ PublicEvaluatorOutputDefinitionRead,
+} from "./publicEvalsContract";
+
+export const PublicApiCreator = z
+ .object({
+ id: z.string(),
+ name: z.string().nullable(),
+ })
+ .strict();
+
+const EvaluatorChatMessage = z
+ .object({
+ role: z.literal("user"),
+ content: z.string(),
+ })
+ .strict();
+
+const EvaluatorChatPrompt = z.array(EvaluatorChatMessage).min(1).max(1);
+const EvaluatorChatPromptInput = z.union([
+ z
+ .string()
+ .min(1)
+ .transform((content) => [{ role: "user" as const, content }]),
+ EvaluatorChatPrompt,
+]);
+
+const EvaluatorVersionBase = z.object({
+ id: z.string(),
+ version: z.number().int().positive(),
+ createdAt: z.coerce.date(),
+ createdBy: PublicApiCreator.nullable(),
+});
+
+const EvaluatorModelConfig = z
+ .object({
+ provider: z.string().min(1),
+ model: z.string().min(1),
+ })
+ .strict();
+
+const LlmAsJudgeEvaluatorVersion = EvaluatorVersionBase.extend({
+ type: z.literal(PUBLIC_EVALUATOR_TYPE_LLM_AS_JUDGE),
+ prompt: EvaluatorChatPrompt,
+ variables: z.array(z.string()),
+ variableMapping: z.array(PromptVariableMappingRead).nullable(),
+ modelConfig: EvaluatorModelConfig.nullable(),
+ outputDefinition: PublicEvaluatorOutputDefinitionRead,
+}).strict();
+
+const CodeEvaluatorVersion = EvaluatorVersionBase.extend({
+ type: z.literal(PUBLIC_EVALUATOR_TYPE_CODE),
+ sourceCode: z.string().min(1),
+ sourceCodeLanguage: PublicCodeEvaluatorSourceCodeLanguage,
+}).strict();
+
+export const EvaluatorVersion = z.discriminatedUnion("type", [
+ LlmAsJudgeEvaluatorVersion,
+ CodeEvaluatorVersion,
+]);
+
+const EvaluationRuleAssignment = z
+ .object({
+ evaluationRuleId: z.string(),
+ variableMappingOverride: z.array(PromptVariableMapping).optional(),
+ })
+ .strict();
+
+const EvaluatorBase = z.object({
+ id: z.string(),
+ name: z.string(),
+ description: z.string().nullable(),
+ createdBy: PublicApiCreator.nullable(),
+ status: z.enum(["active", "paused"]),
+ pausedAt: z.coerce.date().nullable(),
+ pausedReason: z.string().nullable(),
+ pausedMessage: z.string().nullable(),
+ evaluationRuleAssignments: z.array(EvaluationRuleAssignment),
+ createdAt: z.coerce.date(),
+ updatedAt: z.coerce.date(),
+ versionId: z.string(),
+ version: z.number().int().positive(),
+ versionCreatedAt: z.coerce.date(),
+ versionCreatedBy: PublicApiCreator.nullable(),
+});
+
+export const LlmAsJudgeEvaluator = EvaluatorBase.extend({
+ type: z.literal(PUBLIC_EVALUATOR_TYPE_LLM_AS_JUDGE),
+ prompt: EvaluatorChatPrompt,
+ variables: z.array(z.string()),
+ variableMapping: z.array(PromptVariableMappingRead).nullable(),
+ modelConfig: EvaluatorModelConfig.nullable(),
+ outputDefinition: PublicEvaluatorOutputDefinitionRead,
+}).strict();
+
+const CodeEvaluator = EvaluatorBase.extend({
+ type: z.literal(PUBLIC_EVALUATOR_TYPE_CODE),
+ sourceCode: z.string().min(1),
+ sourceCodeLanguage: PublicCodeEvaluatorSourceCodeLanguage,
+}).strict();
+
+export const Evaluator = z.discriminatedUnion("type", [
+ LlmAsJudgeEvaluator,
+ CodeEvaluator,
+]);
+
+const LlmAsJudgeEvaluatorDefinition = z
+ .object({
+ type: z.literal(PUBLIC_EVALUATOR_TYPE_LLM_AS_JUDGE),
+ prompt: EvaluatorChatPromptInput,
+ modelConfig: EvaluatorModelConfig.nullable().optional(),
+ variableMapping: z.array(PromptVariableMappingInput).nullable().optional(),
+ outputDefinition: PublicEvaluatorOutputDefinition,
+ })
+ .strict();
+
+const CodeEvaluatorDefinition = z
+ .object({
+ type: z.literal(PUBLIC_EVALUATOR_TYPE_CODE),
+ sourceCode: z.string().min(1),
+ sourceCodeLanguage: PublicCodeEvaluatorSourceCodeLanguage,
+ })
+ .strict();
+
+const _EvaluatorDefinition = z.discriminatedUnion("type", [
+ LlmAsJudgeEvaluatorDefinition,
+ CodeEvaluatorDefinition,
+]);
+
+const CreateEvaluatorMetadata = z.object({
+ name: z.string().trim().min(1),
+ description: z.string().trim().max(2_000).nullable().optional(),
+});
+
+export const CreateEvaluatorBody = z.discriminatedUnion("type", [
+ CreateEvaluatorMetadata.extend(LlmAsJudgeEvaluatorDefinition.shape).strict(),
+ CreateEvaluatorMetadata.extend(CodeEvaluatorDefinition.shape).strict(),
+]);
+
+const UpdateEvaluatorMetadata = z
+ .object({
+ name: z.string().trim().min(1).optional(),
+ description: z.string().trim().max(2_000).nullable().optional(),
+ })
+ .strict();
+
+export const UpdateEvaluatorBody = z.union([
+ UpdateEvaluatorMetadata.extend(LlmAsJudgeEvaluatorDefinition.shape).strict(),
+ UpdateEvaluatorMetadata.extend(CodeEvaluatorDefinition.shape).strict(),
+ UpdateEvaluatorMetadata.refine((value) => Object.keys(value).length > 0, {
+ message: "Request body cannot be empty",
+ }),
+]);
+
+const decodeCursor = (value: string) => {
+ try {
+ return JSON.parse(Buffer.from(value, "base64url").toString("utf-8"));
+ } catch (_error) {
+ throw new InvalidRequestError("Invalid cursor format");
+ }
+};
+
+const ResourceCursor = z.discriminatedUnion("v", [
+ z.object({
+ v: z.literal(1),
+ lastCreatedAt: z.iso.datetime({ offset: true }),
+ lastId: z.string(),
+ }),
+]);
+
+export type ResourceCursorType = z.infer<typeof ResourceCursor>;
+
+export const EncodedResourceCursor = z
+ .string()
+ .transform(decodeCursor)
+ .pipe(ResourceCursor);
+
+export const EncodedResourceCursorString = z
+ .string()
+ .describe("Opaque cursor for pagination");
+
+export const encodeResourceCursor = (cursor: ResourceCursorType) =>
+ Buffer.from(JSON.stringify(cursor)).toString("base64url");
+
+const EvaluatorVersionCursor = z.discriminatedUnion("v", [
+ z.object({ v: z.literal(1), version: z.number().int().positive() }),
+]);
+
+const EncodedEvaluatorVersionCursor = z
+ .string()
+ .transform(decodeCursor)
+ .pipe(EvaluatorVersionCursor);
+
+export const ListEvaluatorsQuery = z
+ .object({
+ limit: publicApiPaginationLimitZod,
+ cursor: EncodedResourceCursor.optional(),
+ })
+ .strict();
+
+export const ListEvaluatorsResponse = z
+ .object({
+ data: z.array(Evaluator),
+ meta: z.object({ cursor: EncodedResourceCursorString.optional() }).strict(),
+ })
+ .strict();
+
+export const EvaluatorIdQuery = z.object({ evaluatorId: z.string() }).strict();
+
+export const ListEvaluatorVersionsQuery = z
+ .object({
+ evaluatorId: z.string(),
+ limit: publicApiPaginationLimitZod,
+ cursor: EncodedEvaluatorVersionCursor.optional(),
+ })
+ .strict();
+
+export const ListEvaluatorVersionsResponse = z
+ .object({
+ data: z.array(EvaluatorVersion),
+ meta: z.object({ cursor: EncodedResourceCursorString.optional() }).strict(),
+ })
+ .strict();
+
+export const DeleteEvaluatorResponse = z.object({ id: z.string() }).strict();
+
+export type EvaluatorDefinitionType = z.infer<typeof _EvaluatorDefinition>;
+export type CreateEvaluatorBodyType = z.infer<typeof CreateEvaluatorBody>;
+export type UpdateEvaluatorBodyType = z.infer<typeof UpdateEvaluatorBody>;
diff --git a/web/src/features/public-api/types/evaluation/publicEvalsContract.ts b/web/src/features/public-api/types/evaluation/publicEvalsContract.ts
new file mode 100644
index 0000000000..dd90d23a70
--- /dev/null
+++ b/web/src/features/public-api/types/evaluation/publicEvalsContract.ts
@@ -0,0 +1,360 @@
+import {
+ arrayOptionsFilter,
+ categoryOptionsFilter,
+ nullFilter,
+ numberFilter,
+ numberObjectFilter,
+ observationEvalFilterColumns,
+ stringFilter,
+ stringObjectFilter,
+ stringOptionsFilter,
+ timeFilter,
+ booleanFilter,
+ booleanObjectFilter,
+ positionInTraceFilter,
+ langfuseObjects,
+} from "@langfuse/shared";
+import { z } from "zod";
+
+const PUBLIC_EVALUATOR_TYPES = ["llm_as_judge", "code"] as const;
+export const [PUBLIC_EVALUATOR_TYPE_LLM_AS_JUDGE, PUBLIC_EVALUATOR_TYPE_CODE] =
+ PUBLIC_EVALUATOR_TYPES;
+
+const PublicEvaluatorType = z.enum(PUBLIC_EVALUATOR_TYPES);
+export const PublicCodeEvaluatorSourceCodeLanguage = z.enum([
+ "PYTHON",
+ "TYPESCRIPT",
+]);
+
+const _PublicEvaluatorModelConfig = z.object({
+ provider: z.string().min(1),
+ model: z.string().min(1),
+});
+
+const PublicEvaluatorOutputDescription = z.string().trim().min(1).optional();
+
+const PublicEvaluatorOutputDefinitionBase = z.object({
+ scoreReasoningInstructions: PublicEvaluatorOutputDescription,
+ scoreValueInstructions: PublicEvaluatorOutputDescription,
+});
+
+const PublicEvaluatorNumericScore = PublicEvaluatorOutputDefinitionBase.extend({
+ dataType: z.literal("NUMERIC"),
+ minValue: z.number().optional(),
+ maxValue: z.number().optional(),
+}).refine(
+ ({ minValue, maxValue }) =>
+ minValue === undefined || maxValue === undefined || minValue <= maxValue,
+ {
+ message: "Minimum value must be less than or equal to maximum value",
+ },
+);
+
+const PublicEvaluatorBooleanScore = PublicEvaluatorOutputDefinitionBase.extend({
+ dataType: z.literal("BOOLEAN"),
+});
+
+const PublicEvaluatorCategoricalScore =
+ PublicEvaluatorOutputDefinitionBase.extend({
+ dataType: z.literal("CATEGORICAL"),
+ categories: z.array(z.string().trim().min(1)).min(2),
+ shouldAllowMultipleMatches: z.boolean(),
+ });
+
+export const PublicEvaluatorOutputDefinition = z.discriminatedUnion(
+ "dataType",
+ [
+ PublicEvaluatorNumericScore,
+ PublicEvaluatorBooleanScore,
+ PublicEvaluatorCategoricalScore,
+ ],
+);
+
+const PublicEvaluatorOutputDefinitionReadBase = z.object({
+ scoreReasoningInstructions: z.string().optional(),
+ scoreValueInstructions: z.string().optional(),
+});
+
+export const PublicEvaluatorOutputDefinitionRead = z.discriminatedUnion(
+ "dataType",
+ [
+ PublicEvaluatorOutputDefinitionReadBase.extend({
+ dataType: z.literal("NUMERIC"),
+ minValue: z.number().optional(),
+ maxValue: z.number().optional(),
+ }),
+ PublicEvaluatorOutputDefinitionReadBase.extend({
+ dataType: z.literal("BOOLEAN"),
+ }),
+ PublicEvaluatorOutputDefinitionReadBase.extend({
+ dataType: z.literal("CATEGORICAL"),
+ categories: z.array(z.string()),
+ shouldAllowMultipleMatches: z.boolean(),
+ }),
+ ],
+);
+
+const PublicEvaluationRuleTarget = z.enum(["observation", "experiment"]);
+const PublicEvaluationRuleLegacyTarget = z.enum(["trace", "dataset"]);
+const _PublicEvaluationRuleReadTarget = z.union([
+ PublicEvaluationRuleTarget,
+ PublicEvaluationRuleLegacyTarget,
+]);
+
+const _PublicEvaluationRuleStatus = z.enum(["active", "inactive", "paused"]);
+
+// No `scope`: every evaluator is project-owned, so the field could only ever
+// carry one value. Gallery templates are starter definitions that are copied
+// into the project on save, not addressable Langfuse-managed resources.
+const PublicEvaluationRuleEvaluatorReference = z.object({
+ name: z.string().min(1),
+ type: PublicEvaluatorType.default(PUBLIC_EVALUATOR_TYPE_LLM_AS_JUDGE),
+});
+
+// PATCH intentionally omits `type`: an evaluation rule's evaluator type cannot
+// be changed. The service always inherits the rule's current type, so a code
+// rule is never retargeted to an LLM evaluator family, which stores its
+// mapping per assignment and would inherit none. To use a different evaluator
+// type, create a new rule.
+const _PublicEvaluationRuleEvaluator =
+ PublicEvaluationRuleEvaluatorReference.extend({
+ id: z.string(),
+ });
+
+const ObservationPromptVariableMappingSource = z.enum([
+ "input",
+ "output",
+ "metadata",
+ "tool_calls",
+]);
+
+const ExperimentPromptVariableMappingSource = z.enum([
+ "input",
+ "output",
+ "metadata",
+ "tool_calls",
+ "expected_output",
+ "experiment_item_metadata",
+]);
+
+function createMappingSchema<
+ TSource extends z.ZodType<
+ | "input"
+ | "output"
+ | "metadata"
+ | "tool_calls"
+ | "expected_output"
+ | "experiment_item_metadata"
+ >,
+>(sourceSchema: TSource) {
+ return z.object({
+ variable: z.string().min(1),
+ source: sourceSchema,
+ jsonPath: z.string().min(1).optional(),
+ });
+}
+
+const ObservationPromptVariableMappingInput = createMappingSchema(
+ ObservationPromptVariableMappingSource,
+);
+
+const ExperimentPromptVariableMappingInput = createMappingSchema(
+ ExperimentPromptVariableMappingSource,
+);
+
+export const PromptVariableMappingInput = z.union([
+ ObservationPromptVariableMappingInput,
+ ExperimentPromptVariableMappingInput,
+]);
+
+// This shape preserves incomplete mappings so callers can repair them.
+// Mapping inputs remain strict and require a concrete source.
+export const PromptVariableMappingRead = z.object({
+ variable: z.string().min(1),
+ source: ExperimentPromptVariableMappingSource.nullable(),
+ jsonPath: z.string().min(1).optional(),
+});
+
+export const LegacyPromptVariableMapping = z
+ .object({
+ mappingType: z.literal("legacy"),
+ variable: z.string().min(1),
+ langfuseObject: z.enum(langfuseObjects),
+ objectName: z.string().nullable(),
+ source: z.string().min(1),
+ jsonPath: z.string().min(1).optional(),
+ })
+ .refine(
+ (mapping) =>
+ mapping.langfuseObject === "trace" ||
+ mapping.langfuseObject === "dataset_item" ||
+ mapping.objectName !== null,
+ {
+ path: ["objectName"],
+ message: "objectName is required for observation objects",
+ },
+ );
+
+export const PromptVariableMapping = z.union([
+ LegacyPromptVariableMapping,
+ PromptVariableMappingRead,
+]);
+
+const PublicEvaluationRuleReadFilterBase = z
+ .object({
+ type: z.string(),
+ column: z.string(),
+ operator: z.string(),
+ value: z.unknown().optional(),
+ })
+ .loose();
+
+export const PublicEvaluationRuleReadFilter = z.union([
+ PublicEvaluationRuleReadFilterBase.safeExtend({ key: z.string() }),
+ PublicEvaluationRuleReadFilterBase,
+]);
+
+const filterSchemaFactories = {
+ datetime: (columnId: string) =>
+ timeFilter.safeExtend({ column: z.literal(columnId) }),
+ string: (columnId: string) =>
+ stringFilter.safeExtend({ column: z.literal(columnId) }),
+ number: (columnId: string) =>
+ numberFilter.safeExtend({ column: z.literal(columnId) }),
+ stringOptions: (columnId: string) =>
+ stringOptionsFilter.safeExtend({ column: z.literal(columnId) }),
+ categoryOptions: (columnId: string) =>
+ categoryOptionsFilter.safeExtend({ column: z.literal(columnId) }),
+ arrayOptions: (columnId: string) =>
+ arrayOptionsFilter.safeExtend({ column: z.literal(columnId) }),
+ stringObject: (columnId: string) =>
+ stringObjectFilter.safeExtend({ column: z.literal(columnId) }),
+ numberObject: (columnId: string) =>
+ numberObjectFilter.safeExtend({ column: z.literal(columnId) }),
+ boolean: (columnId: string) =>
+ booleanFilter.safeExtend({ column: z.literal(columnId) }),
+ booleanObject: (columnId: string) =>
+ booleanObjectFilter.safeExtend({ column: z.literal(columnId) }),
+ null: (columnId: string) =>
+ nullFilter.safeExtend({ column: z.literal(columnId) }),
+ positionInTrace: (columnId: string) =>
+ positionInTraceFilter.safeExtend({ column: z.literal(columnId) }),
+} as const;
+
+type SupportedFilterFactory = keyof typeof filterSchemaFactories;
+
+function createTargetFilterSchema(
+ columns: Array<{ id: string; type: SupportedFilterFactory }>,
+) {
+ const schemas = columns.map((column) =>
+ filterSchemaFactories[column.type](column.id),
+ );
+
+ if (schemas.length === 1) {
+ return schemas[0]!;
+ }
+
+ return z.union(
+ schemas as [
+ (typeof schemas)[number],
+ (typeof schemas)[number],
+ ...Array<(typeof schemas)[number]>,
+ ],
+ );
+}
+
+// The experiment-root filter is what the `experiment` target *means*, so it is
+// not separately addressable: exposing it on `observation` too would give one
+// rule two contradictory public representations.
+const EXPERIMENT_ROOT_FILTER_COLUMN = "isExperimentItemRootSpan";
+
+const OBSERVATION_EVALUATION_RULE_FILTER_COLUMNS = observationEvalFilterColumns
+ .filter((column) => column.id !== EXPERIMENT_ROOT_FILTER_COLUMN)
+ .map((column) => ({
+ id: column.id,
+ type: column.type,
+ }));
+
+// An experiment rule is an observation rule scoped to experiment root spans,
+// so it accepts every observation filter plus the dataset scope.
+const EXPERIMENT_EVALUATION_RULE_FILTER_COLUMNS = [
+ ...OBSERVATION_EVALUATION_RULE_FILTER_COLUMNS,
+ {
+ id: "datasetId",
+ type: "stringOptions" as const,
+ },
+];
+
+const ObservationEvaluationRuleFilter = createTargetFilterSchema(
+ OBSERVATION_EVALUATION_RULE_FILTER_COLUMNS,
+);
+
+const ExperimentEvaluationRuleFilter = createTargetFilterSchema(
+ EXPERIMENT_EVALUATION_RULE_FILTER_COLUMNS,
+);
+
+const _PublicEvaluationRuleFilter = z.union([
+ ObservationEvaluationRuleFilter,
+ ExperimentEvaluationRuleFilter,
+]);
+
+// Stable evaluation rules are all event rules. Experiment scope is expressed
+// by the isExperimentItemRootSpan filter rather than a separate target field.
+const STABLE_EVALUATION_RULE_FILTER_COLUMNS = [
+ ...observationEvalFilterColumns.map((column) => ({
+ id: column.id,
+ type: column.type,
+ })),
+ {
+ id: "datasetId",
+ type: "stringOptions" as const,
+ },
+];
+
+export const StableEvaluationRuleFilter = createTargetFilterSchema(
+ STABLE_EVALUATION_RULE_FILTER_COLUMNS,
+);
+
+export type PublicEvaluatorModelConfigType = z.infer<
+ typeof _PublicEvaluatorModelConfig
+>;
+export type PublicEvaluatorOutputDefinitionType = z.infer<
+ typeof PublicEvaluatorOutputDefinition
+>;
+export type PublicEvaluatorTypeType = z.infer<typeof PublicEvaluatorType>;
+export type PublicEvaluationRuleTargetType = z.infer<
+ typeof PublicEvaluationRuleTarget
+>;
+export type PublicEvaluationRuleLegacyTargetType = z.infer<
+ typeof PublicEvaluationRuleLegacyTarget
+>;
+export type PublicEvaluationRuleReadTargetType = z.infer<
+ typeof _PublicEvaluationRuleReadTarget
+>;
+export type PublicEvaluationRuleStatusType = z.infer<
+ typeof _PublicEvaluationRuleStatus
+>;
+export type PublicEvaluationRuleEvaluatorReferenceType = z.infer<
+ typeof PublicEvaluationRuleEvaluatorReference
+>;
+export type PublicEvaluationRuleEvaluatorType = z.infer<
+ typeof _PublicEvaluationRuleEvaluator
+>;
+export type PromptVariableMappingInputType = z.infer<
+ typeof PromptVariableMappingInput
+>;
+export type PromptVariableMappingReadType = z.infer<
+ typeof PromptVariableMappingRead
+>;
+export type ObservationPromptVariableMappingInputType = z.infer<
+ typeof ObservationPromptVariableMappingInput
+>;
+export type LegacyPromptVariableMappingType = z.infer<
+ typeof LegacyPromptVariableMapping
+>;
+export type PublicEvaluationRuleFilterType = z.infer<
+ typeof _PublicEvaluationRuleFilter
+>;
+export type PublicCodeEvaluatorSourceCodeLanguageType = z.infer<
+ typeof PublicCodeEvaluatorSourceCodeLanguage
+>;
diff --git a/web/src/features/public-api/types/public-api-errors.ts b/web/src/features/public-api/types/public-api-errors.ts
new file mode 100644
index 0000000000..a246c89c3a
--- /dev/null
+++ b/web/src/features/public-api/types/public-api-errors.ts
@@ -0,0 +1,25 @@
+import { z } from "zod";
+import { StructuredPublicApiErrorDetails } from "./structuredPublicApiErrorSchema";
+
+const publicApiErrorCodes = [
+ "authentication_failed",
+ "access_denied",
+ "invalid_request",
+ "invalid_query",
+ "invalid_body",
+ "resource_not_found",
+ "conflict",
+ "rate_limited",
+ "method_not_allowed",
+ "internal_error",
+] as const;
+
+const PublicApiErrorCode = z.enum(publicApiErrorCodes);
+
+export const PublicApiError = z
+ .object({
+ message: z.string(),
+ code: PublicApiErrorCode,
+ details: StructuredPublicApiErrorDetails.optional(),
+ })
+ .strict();
diff --git a/web/src/features/public-api/types/structuredPublicApiError.ts b/web/src/features/public-api/types/structuredPublicApiError.ts
new file mode 100644
index 0000000000..1ba5a7b01a
--- /dev/null
+++ b/web/src/features/public-api/types/structuredPublicApiError.ts
@@ -0,0 +1,30 @@
+import { BaseError } from "@langfuse/shared";
+import type {
+ StructuredPublicApiErrorCodeType,
+ StructuredPublicApiErrorDetailsType,
+} from "./structuredPublicApiErrorSchema";
+
+export class StructuredPublicApiError extends BaseError {
+ public readonly code: StructuredPublicApiErrorCodeType;
+ public readonly details?: StructuredPublicApiErrorDetailsType;
+
+ constructor(params: {
+ httpCode: number;
+ code: StructuredPublicApiErrorCodeType;
+ message: string;
+ details?: StructuredPublicApiErrorDetailsType;
+ }) {
+ super("StructuredPublicApiError", params.httpCode, params.message, true);
+ this.code = params.code;
+ this.details = params.details;
+ }
+}
+
+export function createStructuredPublicApiError(params: {
+ httpCode: number;
+ code: StructuredPublicApiErrorCodeType;
+ message: string;
+ details?: StructuredPublicApiErrorDetailsType;
+}) {
+ return new StructuredPublicApiError(params);
+}
diff --git a/web/src/features/public-api/shared/unstable-public-api-error-schema.ts b/web/src/features/public-api/types/structuredPublicApiErrorSchema.ts
similarity index 68%
rename from web/src/features/public-api/shared/unstable-public-api-error-schema.ts
rename to web/src/features/public-api/types/structuredPublicApiErrorSchema.ts
index 056aab0cf8..49d39de7f6 100644
--- a/web/src/features/public-api/shared/unstable-public-api-error-schema.ts
+++ b/web/src/features/public-api/types/structuredPublicApiErrorSchema.ts
@@ -1,6 +1,6 @@
import { z } from "zod";
-const unstablePublicApiErrorCodes = [
+const structuredPublicApiErrorCodes = [
"authentication_failed",
"access_denied",
"invalid_request",
@@ -22,9 +22,9 @@ const unstablePublicApiErrorCodes = [
"internal_error",
] as const;
-const UnstablePublicApiErrorCode = z.enum(unstablePublicApiErrorCodes);
+const StructuredPublicApiErrorCode = z.enum(structuredPublicApiErrorCodes);
-const UnstablePublicApiValidationIssue = z
+const StructuredPublicApiValidationIssue = z
.object({
code: z.string(),
message: z.string(),
@@ -32,9 +32,9 @@ const UnstablePublicApiValidationIssue = z
})
.loose();
-const UnstablePublicApiErrorDetails = z
+export const StructuredPublicApiErrorDetails = z
.object({
- issues: z.array(UnstablePublicApiValidationIssue).optional(),
+ issues: z.array(StructuredPublicApiValidationIssue).optional(),
field: z.string().optional(),
column: z.string().optional(),
invalidValues: z.array(z.string()).optional(),
@@ -52,17 +52,17 @@ const UnstablePublicApiErrorDetails = z
})
.strict();
-export const UnstablePublicApiErrorResponse = z
+export const StructuredPublicApiErrorResponse = z
.object({
message: z.string(),
- code: UnstablePublicApiErrorCode,
- details: UnstablePublicApiErrorDetails.optional(),
+ code: StructuredPublicApiErrorCode,
+ details: StructuredPublicApiErrorDetails.optional(),
})
.strict();
-export type UnstablePublicApiErrorCodeType = z.infer<
- typeof UnstablePublicApiErrorCode
+export type StructuredPublicApiErrorCodeType = z.infer<
+ typeof StructuredPublicApiErrorCode
>;
-export type UnstablePublicApiErrorDetailsType = z.infer<
- typeof UnstablePublicApiErrorDetails
+export type StructuredPublicApiErrorDetailsType = z.infer<
+ typeof StructuredPublicApiErrorDetails
>;
diff --git a/web/src/features/public-api/types/unstable-evaluation-rules.ts b/web/src/features/public-api/types/unstable-evaluation-rules.ts
index 22bf83b6a8..69fa6d6fb7 100644
--- a/web/src/features/public-api/types/unstable-evaluation-rules.ts
+++ b/web/src/features/public-api/types/unstable-evaluation-rules.ts
@@ -2,16 +2,16 @@ import { JobTimeScopeZod, singleFilter } from "@langfuse/shared";
import { z } from "zod";
import {
ExperimentEvaluationRuleFilter,
- ExperimentEvaluationRuleMapping,
- LegacyEvaluationRuleMapping,
+ ExperimentPromptVariableMappingInput,
+ LegacyPromptVariableMapping,
ObservationEvaluationRuleFilter,
- ObservationEvaluationRuleMapping,
+ ObservationPromptVariableMappingInput,
PUBLIC_EVALUATOR_TYPE_CODE,
PublicEvaluationRuleFilter,
PublicEvaluationRuleEvaluator,
PublicEvaluationRuleEvaluatorReference,
PublicEvaluationRuleEvaluatorReferencePatch,
- PublicEvaluationRuleReadMapping,
+ PromptVariableMappingRead,
PublicEvaluationRuleLegacyTarget,
PublicEvaluationRuleStatus,
PublicEvaluationRuleTarget,
@@ -39,12 +39,12 @@ const APIEvaluationRule = z
evaluators: z.array(
z.object({
evaluator: PublicEvaluationRuleEvaluator,
- mapping: z.array(PublicEvaluationRuleReadMapping).nullable(),
+ mapping: z.array(PromptVariableMappingRead).nullable(),
}),
),
target: PublicEvaluationRuleTarget,
filter: z.array(PublicEvaluationRuleFilter),
- mapping: z.array(PublicEvaluationRuleReadMapping),
+ mapping: z.array(PromptVariableMappingRead),
})
.strict();
@@ -54,14 +54,14 @@ const APILegacyEvaluationRule = z
evaluators: z.array(
z.object({
evaluator: PublicEvaluationRuleEvaluator,
- mapping: z.array(LegacyEvaluationRuleMapping).nullable(),
+ mapping: z.array(LegacyPromptVariableMapping).nullable(),
}),
),
target: PublicEvaluationRuleLegacyTarget,
delay: z.number().int().nonnegative(),
timeScope: z.array(JobTimeScopeZod),
filter: z.array(singleFilter),
- mapping: z.array(LegacyEvaluationRuleMapping),
+ mapping: z.array(LegacyPromptVariableMapping),
})
.strict();
@@ -103,25 +103,25 @@ function evaluatorAssignment<T extends z.ZodType>(mapping: T) {
const PostUnstableObservationEvaluationRuleBody = z.object({
...EvaluationRuleCreateBase,
evaluators: z
- .array(evaluatorAssignment(ObservationEvaluationRuleMapping))
+ .array(evaluatorAssignment(ObservationPromptVariableMappingInput))
.min(1)
.max(100)
.optional(),
target: z.literal("observation"),
filter: z.array(ObservationEvaluationRuleFilter).default([]),
- mapping: z.array(ObservationEvaluationRuleMapping).optional(),
+ mapping: z.array(ObservationPromptVariableMappingInput).optional(),
});
const PostUnstableExperimentEvaluationRuleBody = z.object({
...EvaluationRuleCreateBase,
evaluators: z
- .array(evaluatorAssignment(ExperimentEvaluationRuleMapping))
+ .array(evaluatorAssignment(ExperimentPromptVariableMappingInput))
.min(1)
.max(100)
.optional(),
target: z.literal("experiment"),
filter: z.array(ExperimentEvaluationRuleFilter).default([]),
- mapping: z.array(ExperimentEvaluationRuleMapping).optional(),
+ mapping: z.array(ExperimentPromptVariableMappingInput).optional(),
});
// `code` evaluators use a fixed runtime mapping managed by Langfuse. LLM
@@ -196,7 +196,7 @@ const EXCLUSIVE_EVALUATOR_FIELDS_MESSAGE =
const UntargetedEvaluationRulePatch = z.object({
...EvaluationRulePatchBase,
- evaluators: patchEvaluators(ObservationEvaluationRuleMapping),
+ evaluators: patchEvaluators(ObservationPromptVariableMappingInput),
target: z.undefined().optional(),
filter: z.undefined().optional(),
mapping: z.undefined().optional(),
@@ -204,18 +204,18 @@ const UntargetedEvaluationRulePatch = z.object({
const ObservationEvaluationRulePatch = z.object({
...EvaluationRulePatchBase,
- evaluators: patchEvaluators(ObservationEvaluationRuleMapping),
+ evaluators: patchEvaluators(ObservationPromptVariableMappingInput),
target: z.literal("observation"),
filter: z.array(ObservationEvaluationRuleFilter).optional(),
- mapping: z.array(ObservationEvaluationRuleMapping).optional(),
+ mapping: z.array(ObservationPromptVariableMappingInput).optional(),
});
const ExperimentEvaluationRulePatch = z.object({
...EvaluationRulePatchBase,
- evaluators: patchEvaluators(ExperimentEvaluationRuleMapping),
+ evaluators: patchEvaluators(ExperimentPromptVariableMappingInput),
target: z.literal("experiment"),
filter: z.array(ExperimentEvaluationRuleFilter).optional(),
- mapping: z.array(ExperimentEvaluationRuleMapping).optional(),
+ mapping: z.array(ExperimentPromptVariableMappingInput).optional(),
});
export const PatchUnstableEvaluationRuleBody = z
diff --git a/web/src/features/public-api/types/unstable-evaluators.ts b/web/src/features/public-api/types/unstable-evaluators.ts
index 3ea2076a36..74779a0788 100644
--- a/web/src/features/public-api/types/unstable-evaluators.ts
+++ b/web/src/features/public-api/types/unstable-evaluators.ts
@@ -2,8 +2,8 @@ import { z } from "zod";
import {
PUBLIC_EVALUATOR_TYPE_CODE,
PUBLIC_EVALUATOR_TYPE_LLM_AS_JUDGE,
- ObservationEvaluationRuleMapping,
- PublicEvaluationRuleReadMapping,
+ ObservationPromptVariableMappingInput,
+ PromptVariableMappingRead,
PublicCodeEvaluatorDefinitionInput,
PublicEvaluatorModelConfig,
PublicEvaluatorOutputDefinition,
@@ -20,7 +20,7 @@ const APIEvaluatorBase = z
variables: z.array(z.string()),
// An evaluator's default mapping can name experiment-only sources, and a legacy one can be
// incomplete, so reads use the permissive schema. Requests stay strict.
- mapping: z.array(PublicEvaluationRuleReadMapping).nullable(),
+ mapping: z.array(PromptVariableMappingRead).nullable(),
evaluationRuleCount: z.number().int().nonnegative(),
createdAt: z.coerce.date(),
updatedAt: z.coerce.date(),
@@ -81,7 +81,7 @@ const PostUnstableLlmAsJudgeEvaluatorBody = z.object({
...EvaluatorCreateBase,
type: z.literal(PUBLIC_EVALUATOR_TYPE_LLM_AS_JUDGE),
...PublicLlmAsJudgeEvaluatorDefinitionInput.shape,
- mapping: z.array(ObservationEvaluationRuleMapping).optional(),
+ mapping: z.array(ObservationPromptVariableMappingInput).optional(),
sourceCode: z.never().optional(),
sourceCodeLanguage: z.never().optional(),
});
diff --git a/web/src/features/public-api/types/unstable-public-evals-contract.ts b/web/src/features/public-api/types/unstable-public-evals-contract.ts
index ce26632e84..09fdab098b 100644
--- a/web/src/features/public-api/types/unstable-public-evals-contract.ts
+++ b/web/src/features/public-api/types/unstable-public-evals-contract.ts
@@ -12,15 +12,15 @@ import {
timeFilter,
experimentEvalFilterColumns,
booleanFilter,
+ CODE_EVAL_SOURCE_MAX_BYTES,
langfuseObjects,
} from "@langfuse/shared";
-import { CODE_EVAL_SOURCE_MAX_BYTES } from "@langfuse/shared/src/server";
import { z } from "zod";
-export { UnstablePublicApiErrorResponse } from "@/src/features/public-api/shared/unstable-public-api-error-schema";
+export { StructuredPublicApiErrorResponse } from "./structuredPublicApiErrorSchema";
import type {
- UnstablePublicApiErrorCodeType,
- UnstablePublicApiErrorDetailsType,
-} from "@/src/features/public-api/shared/unstable-public-api-error-schema";
+ StructuredPublicApiErrorCodeType,
+ StructuredPublicApiErrorDetailsType,
+} from "./structuredPublicApiErrorSchema";
const PUBLIC_EVALUATOR_TYPES = ["llm_as_judge", "code"] as const;
export const [PUBLIC_EVALUATOR_TYPE_LLM_AS_JUDGE, PUBLIC_EVALUATOR_TYPE_CODE] =
@@ -85,9 +85,7 @@ export const PublicEvaluatorOutputDefinition = z.discriminatedUnion(
export const PublicEvaluationRuleTarget = z.enum(["observation", "experiment"]);
export const PublicEvaluationRuleLegacyTarget = z.enum(["trace", "dataset"]);
-
-// eslint-disable-next-line @typescript-eslint/no-unused-vars -- Used via z.infer
-const PublicEvaluationRuleReadTarget = z.union([
+const _PublicEvaluationRuleReadTarget = z.union([
PublicEvaluationRuleTarget,
PublicEvaluationRuleLegacyTarget,
]);
@@ -120,14 +118,14 @@ export const PublicEvaluationRuleEvaluator =
id: z.string(),
});
-export const ObservationEvaluationRuleMappingSource = z.enum([
+export const ObservationPromptVariableMappingSource = z.enum([
"input",
"output",
"metadata",
"tool_calls",
]);
-export const ExperimentEvaluationRuleMappingSource = z.enum([
+export const ExperimentPromptVariableMappingSource = z.enum([
"input",
"output",
"metadata",
@@ -153,29 +151,28 @@ function createMappingSchema<
});
}
-export const ObservationEvaluationRuleMapping = createMappingSchema(
- ObservationEvaluationRuleMappingSource,
+export const ObservationPromptVariableMappingInput = createMappingSchema(
+ ObservationPromptVariableMappingSource,
);
-export const ExperimentEvaluationRuleMapping = createMappingSchema(
- ExperimentEvaluationRuleMappingSource,
+export const ExperimentPromptVariableMappingInput = createMappingSchema(
+ ExperimentPromptVariableMappingSource,
);
-// eslint-disable-next-line @typescript-eslint/no-unused-vars -- Used via z.infer
-const PublicEvaluationRuleMapping = z.union([
- ObservationEvaluationRuleMapping,
- ExperimentEvaluationRuleMapping,
+const _PromptVariableMappingInput = z.union([
+ ObservationPromptVariableMappingInput,
+ ExperimentPromptVariableMappingInput,
]);
-// Read responses preserve incomplete legacy mappings so callers can repair
-// them. Write schemas remain strict and require a concrete source.
-export const PublicEvaluationRuleReadMapping = z.object({
+// This shape preserves incomplete mappings so callers can repair them.
+// Mapping inputs remain strict and require a concrete source.
+export const PromptVariableMappingRead = z.object({
variable: z.string().min(1),
- source: ExperimentEvaluationRuleMappingSource.nullable(),
+ source: ExperimentPromptVariableMappingSource.nullable(),
jsonPath: z.string().min(1).optional(),
});
-export const LegacyEvaluationRuleMapping = z
+export const LegacyPromptVariableMapping = z
.object({
variable: z.string().min(1),
langfuseObject: z.enum(langfuseObjects),
@@ -289,7 +286,7 @@ export type PublicEvaluationRuleLegacyTargetType = z.infer<
typeof PublicEvaluationRuleLegacyTarget
>;
export type PublicEvaluationRuleReadTargetType = z.infer<
- typeof PublicEvaluationRuleReadTarget
+ typeof _PublicEvaluationRuleReadTarget
>;
export type PublicEvaluationRuleStatusType = z.infer<
typeof PublicEvaluationRuleStatus
@@ -300,24 +297,24 @@ export type PublicEvaluationRuleEvaluatorReferenceType = z.infer<
export type PublicEvaluationRuleEvaluatorType = z.infer<
typeof PublicEvaluationRuleEvaluator
>;
-export type PublicEvaluationRuleMappingType = z.infer<
- typeof PublicEvaluationRuleMapping
+export type PromptVariableMappingInputType = z.infer<
+ typeof _PromptVariableMappingInput
>;
-export type PublicEvaluationRuleReadMappingType = z.infer<
- typeof PublicEvaluationRuleReadMapping
+export type PromptVariableMappingReadType = z.infer<
+ typeof PromptVariableMappingRead
>;
-export type PublicObservationEvaluationRuleMappingType = z.infer<
- typeof ObservationEvaluationRuleMapping
+export type ObservationPromptVariableMappingInputType = z.infer<
+ typeof ObservationPromptVariableMappingInput
>;
-export type LegacyEvaluationRuleMappingType = z.infer<
- typeof LegacyEvaluationRuleMapping
+export type LegacyPromptVariableMappingType = z.infer<
+ typeof LegacyPromptVariableMapping
>;
export type PublicEvaluationRuleFilterType = z.infer<
typeof PublicEvaluationRuleFilter
>;
export type {
- UnstablePublicApiErrorCodeType,
- UnstablePublicApiErrorDetailsType,
+ StructuredPublicApiErrorCodeType,
+ StructuredPublicApiErrorDetailsType,
};
export const UnstablePublicApiPaginationQuery = z.object({
diff --git a/web/src/features/widgets/server/public-dashboard-widget-service.ts b/web/src/features/widgets/server/public-dashboard-widget-service.ts
index 7ea9587baf..d6a6fea15e 100644
--- a/web/src/features/widgets/server/public-dashboard-widget-service.ts
+++ b/web/src/features/widgets/server/public-dashboard-widget-service.ts
@@ -14,7 +14,7 @@ import {
type ViewVersion,
} from "@langfuse/shared/query";
import { auditLog } from "@/src/features/audit-logs/auditLog";
-import { createUnstablePublicApiError } from "@/src/features/public-api/server/unstable-public-api-error-contract";
+import { createStructuredPublicApiError } from "@/src/features/public-api";
import {
PostUnstableDashboardWidgetResponse,
type DashboardWidgetViewOutputType,
@@ -42,7 +42,7 @@ const throwInvalidWidget = (params: {
field?: string;
allowedValues?: string[];
}): never => {
- throw createUnstablePublicApiError({
+ throw createStructuredPublicApiError({
httpCode: 400,
code: "invalid_request",
message: params.message,
diff --git a/web/src/pages/api/public/v2/evaluation-rules/[evaluationRuleId].ts b/web/src/pages/api/public/v2/evaluation-rules/[evaluationRuleId].ts
new file mode 100644
index 0000000000..64c1ac60f9
--- /dev/null
+++ b/web/src/pages/api/public/v2/evaluation-rules/[evaluationRuleId].ts
@@ -0,0 +1 @@
+export { evaluationRuleApiHandler as default } from "@/src/features/public-api/server";
diff --git a/web/src/pages/api/public/v2/evaluation-rules/index.ts b/web/src/pages/api/public/v2/evaluation-rules/index.ts
new file mode 100644
index 0000000000..336cda3b18
--- /dev/null
+++ b/web/src/pages/api/public/v2/evaluation-rules/index.ts
@@ -0,0 +1 @@
+export { evaluationRulesApiHandler as default } from "@/src/features/public-api/server";
diff --git a/web/src/pages/api/public/v2/evaluators/[evaluatorId].ts b/web/src/pages/api/public/v2/evaluators/[evaluatorId].ts
new file mode 100644
index 0000000000..6f7f385879
--- /dev/null
+++ b/web/src/pages/api/public/v2/evaluators/[evaluatorId].ts
@@ -0,0 +1 @@
+export { evaluatorApiHandler as default } from "@/src/features/public-api/server";
diff --git a/web/src/pages/api/public/v2/evaluators/[evaluatorId]/versions.ts b/web/src/pages/api/public/v2/evaluators/[evaluatorId]/versions.ts
new file mode 100644
index 0000000000..ce6ce09921
--- /dev/null
+++ b/web/src/pages/api/public/v2/evaluators/[evaluatorId]/versions.ts
@@ -0,0 +1 @@
+export { evaluatorVersionsApiHandler as default } from "@/src/features/public-api/server";
diff --git a/web/src/pages/api/public/v2/evaluators/index.ts b/web/src/pages/api/public/v2/evaluators/index.ts
new file mode 100644
index 0000000000..632118bf32
--- /dev/null
+++ b/web/src/pages/api/public/v2/evaluators/index.ts
@@ -0,0 +1 @@
+export { evaluatorsApiHandler as default } from "@/src/features/public-api/server";