🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(api): add stable evaluator and evaluation rule endpoints (#16608)
Tobias Wochinger96d0ce458689
diff --git a/fern/apis/server/definition/evaluation-commons.yml b/fern/apis/server/definition/evaluation-commons.ymlnew file mode 100644index 0000000000..2c43010194--- /dev/null+++ b/fern/apis/server/definition/evaluation-commons.yml@@ -0,0 +1,606 @@+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json+types:+ EvaluatorType:+ docs: |+ The evaluator type.++ The public API supports LLM-as-a-judge and code evaluators.+ enum:+ - llm_as_judge+ - code++ CodeEvaluatorSourceCodeLanguage:+ docs: Code evaluator runtime language.+ enum:+ - PYTHON+ - TYPESCRIPT++ PromptVariableMappingSource:+ docs: |+ Source field used to populate a prompt variable.++ Use these values when mapping evaluator prompt variables to live data.++ Source semantics:+ - `input`: the observation input payload+ - `output`: the observation output payload+ - `metadata`: the observation metadata object. Combine with `jsonPath` when you need one nested field instead of the whole object.+ - `tool_calls`: the tool calls recorded on the observation, as an array of `{id, name, arguments, type, index}` objects in the order the model emitted them. Combine with `jsonPath` (for example `$[*].name`) to select parts of each call.+ - `expected_output`: the experiment item's expected output when the observation belongs to an experiment.+ - `experiment_item_metadata`: the experiment item's metadata when the observation belongs to an experiment.+ enum:+ - input+ - output+ - metadata+ - tool_calls+ - expected_output+ - experiment_item_metadata++ EvaluatorOutputScoreType:+ docs: |+ Structured score type returned by an evaluator.++ This controls the type of score value Langfuse stores for evaluation results:+ - `NUMERIC`: a numeric score such as `0.82`+ - `BOOLEAN`: a boolean score such as `true`+ - `CATEGORICAL`: one or more category labels from a fixed list+ enum:+ - NUMERIC+ - BOOLEAN+ - CATEGORICAL++ EvaluatorOutputDefinition:+ docs: |+ Flat structured output definition used when creating or updating an evaluator.++ - `dataType` is required.+ - `scoreReasoningInstructions` and `scoreValueInstructions` are optional instructions.+ - `minValue` and `maxValue` apply only to `NUMERIC` outputs. If both are set, `minValue` must not exceed `maxValue`.+ - `categories` and `shouldAllowMultipleMatches` apply only to `CATEGORICAL` outputs.+ - Do not send `version`; that is an internal storage detail.+ discriminant: dataType+ union:+ NUMERIC:+ type: PublicEvaluatorNumericScore+ BOOLEAN:+ type: PublicEvaluatorBooleanScore+ CATEGORICAL:+ type: PublicEvaluatorCategoricalScore+ examples:+ - name: Numeric+ value:+ dataType: NUMERIC+ minValue: 0+ maxValue: 1+ - name: Boolean+ value:+ dataType: BOOLEAN+ scoreValueInstructions: Return true if the output satisfies the requirement, otherwise false.+ - name: Categorical+ value:+ dataType: CATEGORICAL+ scoreReasoningInstructions: Explain which category best fits the output.+ scoreValueInstructions: Choose the best category.+ categories:+ - correct+ - partially_correct+ - incorrect+ shouldAllowMultipleMatches: false++ EvaluatorOutputDefinitionBase:+ properties:+ scoreReasoningInstructions:+ type: optional<string>+ docs: Optional instructions for deriving the reasoning returned with the score.+ scoreValueInstructions:+ type: optional<string>+ docs: Optional instructions for deriving the score value.++ PublicEvaluatorNumericScore:+ extends: EvaluatorOutputDefinitionBase+ properties:+ dataType:+ type: literal<"NUMERIC">+ docs: Numeric score output.+ minValue:+ type: optional<double>+ docs: Optional inclusive minimum value.+ maxValue:+ type: optional<double>+ docs: Optional inclusive maximum value.++ PublicEvaluatorBooleanScore:+ extends: EvaluatorOutputDefinitionBase+ properties:+ dataType:+ type: literal<"BOOLEAN">+ docs: Boolean score output.++ PublicEvaluatorCategoricalScore:+ extends: EvaluatorOutputDefinitionBase+ properties:+ dataType:+ type: literal<"CATEGORICAL">+ docs: Categorical score output.+ categories:+ type: list<string>+ docs: Allowed category values. At least two unique values are required.+ shouldAllowMultipleMatches:+ type: boolean+ docs: Whether the evaluator may return more than one category.++ PublicEvaluatorOutputDefinition:+ docs: |+ Flat evaluator output definition returned by the public API.++ This response always includes `dataType` and never includes an internal output-definition `version`.+ Optional empty descriptions from legacy definitions are omitted.+ discriminant: dataType+ union:+ NUMERIC:+ type: PublicEvaluatorNumericScore+ BOOLEAN:+ type: PublicEvaluatorBooleanScore+ CATEGORICAL:+ type: PublicEvaluatorCategoricalScore+ examples:+ - name: PublicNumeric+ value:+ dataType: NUMERIC+ minValue: 0+ maxValue: 1+ - name: PublicCategorical+ value:+ dataType: CATEGORICAL+ scoreReasoningInstructions: Explain which label best fits the output.+ scoreValueInstructions: Choose the best label.+ categories:+ - correct+ - partially_correct+ - incorrect+ shouldAllowMultipleMatches: false++ EvaluationRuleStringFilterOperator:+ enum:+ - name: Equals+ value: "="+ - name: Contains+ value: contains+ - name: DoesNotContain+ value: does not contain+ - name: StartsWith+ value: starts with+ - name: EndsWith+ value: ends with++ EvaluationRuleNumberFilterOperator:+ enum:+ - name: Equals+ value: "="+ - name: GreaterThan+ value: ">"+ - name: LessThan+ value: "<"+ - name: GreaterThanOrEqual+ value: ">="+ - name: LessThanOrEqual+ value: "<="++ EvaluationRuleOptionsFilterOperator:+ enum:+ - name: AnyOf+ value: any of+ - name: NoneOf+ value: none of++ EvaluationRuleArrayOptionsFilterOperator:+ enum:+ - name: AnyOf+ value: any of+ - name: NoneOf+ value: none of+ - name: AllOf+ value: all of++ EvaluationRuleBooleanFilterOperator:+ enum:+ - name: Equals+ value: "="+ - name: NotEquals+ value: "<>"++ EvaluationRuleNullFilterOperator:+ enum:+ - name: IsNull+ value: is null+ - name: IsNotNull+ value: is not null++ DateTimeEvaluationRuleFilter:+ properties:+ column:+ type: string+ docs: Column to filter on.+ operator:+ type: EvaluationRuleNumberFilterOperator+ docs: Comparison operator for datetime values.+ value:+ type: datetime+ docs: Datetime value to compare against.++ StringEvaluationRuleFilter:+ properties:+ column:+ type: string+ docs: Column to filter on.+ operator:+ type: EvaluationRuleStringFilterOperator+ value:+ type: string++ NumberEvaluationRuleFilter:+ properties:+ column:+ type: string+ docs: Column to filter on.+ operator:+ type: EvaluationRuleNumberFilterOperator+ value:+ type: double++ StringOptionsEvaluationRuleFilter:+ properties:+ column:+ type: string+ docs: Column to filter on.+ operator:+ type: EvaluationRuleOptionsFilterOperator+ value:+ type: list<string>+ docs: One or more allowed string values.++ ArrayOptionsEvaluationRuleFilter:+ properties:+ column:+ type: string+ docs: Column to filter on.+ operator:+ type: EvaluationRuleArrayOptionsFilterOperator+ value:+ type: list<string>+ docs: One or more array elements to match.++ StringObjectEvaluationRuleFilter:+ properties:+ column:+ type: string+ docs: Object-valued column to filter on. Currently only `metadata` is supported.+ key:+ type: string+ docs: Top-level key inside the object-valued column to filter on.+ operator:+ type: EvaluationRuleStringFilterOperator+ value:+ type: string++ NumberObjectEvaluationRuleFilter:+ properties:+ column:+ type: string+ docs: Object-valued column to filter on.+ key:+ type: string+ docs: Key inside the object-valued column to filter on.+ operator:+ type: EvaluationRuleNumberFilterOperator+ value:+ type: double++ CategoryOptionsEvaluationRuleFilter:+ properties:+ column:+ type: string+ docs: Object-valued column to filter on.+ key:+ type: string+ docs: Key inside the object-valued column to filter on.+ operator:+ type: EvaluationRuleOptionsFilterOperator+ value:+ type: list<string>++ BooleanEvaluationRuleFilter:+ properties:+ column:+ type: string+ docs: Column to filter on.+ operator:+ type: EvaluationRuleBooleanFilterOperator+ value:+ type: boolean++ NullEvaluationRuleFilter:+ properties:+ column:+ type: string+ docs: Column to filter on, for example `parentObservationId`.+ operator:+ type: EvaluationRuleNullFilterOperator+ value:+ type: literal<"">+ docs: Required empty-string placeholder used by the runtime filter contract.++ PromptVariableMappingInput:+ docs: |+ Connects one prompt variable to data from an observation or experiment.++ Manual mappings are used for `llm_as_judge` evaluators. `code` evaluators use a fixed runtime mapping managed by Langfuse.++ How to build a valid mapping list:+ 1. Create the evaluator or fetch it with `GET /evaluators/{id}`.+ 2. Read the evaluator `variables` array.+ 3. Add exactly one mapping object for each variable in that array.+ 4. Use the variable name exactly as returned, without braces such as `{{` or `}}`.+ 5. Choose the source field that should populate the variable.++ `jsonPath` is optional. Use it only when the selected source is a JSON object and you want to extract one nested field before inserting it into the evaluator prompt.++ Invalid, missing, or duplicate mappings return a validation error. Malformed JSONPath expressions are also rejected.+ properties:+ variable:+ type: string+ docs: |+ Prompt variable name without braces.++ Example: for the prompt `Judge {{input}} against {{output}}`, use `input` and `output`.+ source:+ type: PromptVariableMappingSource+ docs: |+ Source field that should populate the prompt variable.++ Available sources are `input`, `output`, `metadata`, `tool_calls`, `expected_output`, and `experiment_item_metadata`.+ jsonPath:+ type: optional<string>+ docs: |+ Optional JSONPath selector applied to the selected source before it is passed to the evaluator prompt.++ Requirements:+ - Must start with `$`+ - Must be a syntactically valid JSONPath expression+ - Most useful with `source=metadata`+ examples:+ - name: BasicObservationMapping+ value:+ variable: input+ source: input+ - name: MetadataProjectionMapping+ value:+ variable: customer_tier+ source: metadata+ jsonPath: "$.customer.tier"+ - name: ExperimentExpectedOutputMapping+ value:+ variable: expected_output+ source: expected_output++ PromptVariableMappingRead:+ docs: |+ Connects one prompt variable to source data.++ `source` is `null` when the mapping is incomplete.+ properties:+ variable:+ type: string+ docs: Prompt variable name without braces.+ source:+ type: nullable<string>+ docs: Stored source field populating the variable, or `null` when the mapping is incomplete.+ jsonPath:+ type: optional<string>+ docs: Optional JSONPath selector applied to the selected source.+ examples:+ - name: ConfiguredReadMapping+ value:+ variable: input+ source: input+ - name: IncompleteReadMapping+ value:+ variable: input+ source: null++ PromptVariableMapping:+ docs: Connects one prompt variable to source data.+ discriminated: false+ union:+ - PromptVariableMappingRead+ - LegacyPromptVariableMapping++ LegacyPromptVariableMapping:+ docs: |+ **Deprecated:** Connects one prompt variable to data from a legacy trace or dataset evaluation rule.++ `langfuseObject` selects the object kind. `objectName` separately selects a named observation and is `null` for trace and dataset-item mappings.+ properties:+ mappingType:+ type: literal<"legacy">+ docs: Explicitly marks this as a legacy mapping.+ variable:+ type: string+ docs: Prompt variable name without braces.+ langfuseObject:+ type: LegacyEvaluationObject+ docs: Legacy object kind selected as the mapping source.+ objectName:+ type: nullable<string>+ docs: Observation name to match, or `null` when `langfuseObject` is `trace` or `dataset_item`.+ source:+ type: string+ docs: Field selected from the legacy object.+ jsonPath:+ type: optional<string>+ docs: Optional JSONPath selector applied to the selected field.+ examples:+ - name: LegacyTraceInput+ value:+ mappingType: legacy+ variable: input+ langfuseObject: trace+ objectName: null+ source: input+ - name: LegacyNamedGenerationOutput+ value:+ mappingType: legacy+ variable: output+ langfuseObject: generation+ objectName: answer-generation+ source: output+ jsonPath: "$.answer"++ LegacyEvaluationObject:+ docs: "**Deprecated:** Legacy Langfuse object kind used by trace and dataset evaluation rules."+ enum:+ - trace+ - span+ - generation+ - event+ - agent+ - tool+ - chain+ - retriever+ - evaluator+ - embedding+ - guardrail+ - dataset_item++ EvaluationRuleFilter:+ docs: |+ One filter condition used to decide whether a live-ingested observation should be evaluated.++ All filters must be satisfied for the rule to run. Experiment scope is expressed with filters rather than a separate target field:+ - `isExperimentItemRootSpan = true` limits execution to experiment item roots.+ - `datasetId` limits execution to experiments for the selected datasets. Use dataset IDs from `GET /api/public/v2/datasets`.++ Pick the filter `type` first, as it determines the required fields and value shape. Use `key` only for object filters such as `metadata`.++ Operator quick reference by filter `type`:+ - `string`: `"="`, `contains`, `does not contain`, `starts with`, `ends with`+ - `number`: `"="`, `">"`, `"<"`, `">="`, `"<="`+ - `datetime`: `"="`, `">"`, `"<"`, `">="`, `"<="`+ - `stringOptions`: `any of`, `none of`+ - `arrayOptions`: `any of`, `none of`, `all of`+ - `stringObject`: same operators as `string`+ - `boolean`: `"="`, `"<>"`+ - `null`: `is null`, `is not null`++ Supported columns. Each column accepts exactly one filter `type`; sending a column with a different `type` is rejected with `400`.+ - `type`: `stringOptions`, values `SPAN`, `EVENT`, `GENERATION`, `AGENT`, `TOOL`, `CHAIN`, `RETRIEVER`, `EVALUATOR`, `EMBEDDING`, `GUARDRAIL`+ - `name`: `stringOptions`+ - `environment`: `stringOptions`+ - `level`: `stringOptions`, values `DEBUG`, `DEFAULT`, `WARNING`, `ERROR`+ - `version`: `string`+ - `traceName`: `stringOptions`+ - `userId`: `string`+ - `sessionId`: `string`+ - `tags`: `arrayOptions`+ - `metadata`: `stringObject`, requires `key`+ - `isRootObservation`: `boolean`; true when the observation has no parent or is explicitly marked as an application root+ - `parentObservationId`: `null`+ - `experimentId`: `stringOptions`+ - `isExperimentItemRootSpan`: `boolean`+ - `calledToolNames`: `arrayOptions`+ - `toolCalls`: `number`, the number of tool calls on the observation+ - `datasetId`: `stringOptions`. Use dataset `id` values from `GET /api/public/v2/datasets`, not dataset names.+ discriminant: type+ union:+ "datetime":+ type: DateTimeEvaluationRuleFilter+ "string":+ type: StringEvaluationRuleFilter+ "number":+ type: NumberEvaluationRuleFilter+ "stringOptions":+ type: StringOptionsEvaluationRuleFilter+ "categoryOptions":+ type: CategoryOptionsEvaluationRuleFilter+ "arrayOptions":+ type: ArrayOptionsEvaluationRuleFilter+ "stringObject":+ type: StringObjectEvaluationRuleFilter+ "numberObject":+ type: NumberObjectEvaluationRuleFilter+ "boolean":+ type: BooleanEvaluationRuleFilter+ "null":+ type: NullEvaluationRuleFilter+ examples:+ - name: ObservationTypeFilter+ value:+ type: stringOptions+ column: type+ operator: any of+ value:+ - GENERATION+ - name: ObservationMetadataFilter+ value:+ type: stringObject+ column: metadata+ key: customerTier+ operator: "="+ value: enterprise+ - name: ObservationRootOnlyFilter+ value:+ type: "null"+ column: parentObservationId+ operator: is null+ value: ""+ - name: ObservationLogicalRootFilter+ value:+ type: boolean+ column: isRootObservation+ operator: "="+ value: true+ - name: ObservationTagsFilter+ value:+ type: arrayOptions+ column: tags+ operator: any of+ value:+ - production+ - name: ExperimentDatasetFilter+ value:+ type: stringOptions+ column: datasetId+ operator: any of+ value:+ - "550e8400-e29b-41d4-a716-446655440000"++ EvaluationRuleReadFilterBase:+ docs: Stored filter returned verbatim by evaluation-rule read endpoints.+ properties:+ type:+ type: string+ docs: Stored filter type. This is not broken down into separate public response types.+ column:+ type: string+ operator:+ type: string+ value:+ type: optional<unknown>+ docs: Stored filter value. Its shape depends on the filter type.++ EvaluationRuleReadFilterWithKey:+ extends: EvaluationRuleReadFilterBase+ properties:+ key:+ type: string+ docs: Stored object or legacy filter key.++ EvaluationRuleReadFilter:+ docs: |+ Filter returned verbatim by evaluation-rule read endpoints.++ Filters with a stored `key` use the keyed shape. All other filters use the base shape. Response filters are intentionally not broken down by their internal `type` value.+ discriminated: false+ union:+ - EvaluationRuleReadFilterWithKey+ - EvaluationRuleReadFilterBasediff --git a/fern/apis/server/definition/evaluation-errors.yml b/fern/apis/server/definition/evaluation-errors.ymlnew file mode 100644index 0000000000..0e514da84b--- /dev/null+++ b/fern/apis/server/definition/evaluation-errors.yml@@ -0,0 +1,125 @@+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json+types:+ PublicApiErrorCode:+ docs: Stable machine-readable error code returned by the evaluators and evaluation-rules API.+ enum:+ - authentication_failed+ - access_denied+ - invalid_request+ - invalid_query+ - invalid_body+ - resource_not_found+ - conflict+ - rate_limited+ - method_not_allowed+ - internal_error++ PublicApiValidationIssue:+ docs: One request validation issue.+ properties:+ code:+ type: string+ docs: Validator issue code.+ message:+ type: string+ docs: Human-readable explanation.+ path:+ type: list<unknown>+ docs: Path to the invalid request field.++ PublicApiErrorDetails:+ docs: Optional structured context for validation and rate-limit errors.+ properties:+ issues:+ type: optional<list<PublicApiValidationIssue>>+ docs: Validation issues for an invalid request body or query.+ retryAfterSeconds:+ type: optional<integer>+ docs: Number of seconds to wait before retrying a rate-limited request.+ limit:+ type: optional<integer>+ docs: Rate-limit request allowance.+ remaining:+ type: optional<integer>+ docs: Remaining requests in the current rate-limit window.+ resetAt:+ type: optional<datetime>+ docs: Time when the current rate-limit window resets.++ PublicApiError:+ docs: |+ Standard error response for the stable evaluators and evaluation-rules API.++ Use the HTTP status for the broad failure class and `code` for programmatic handling. `details` is included when field-level validation or retry information is available.+ properties:+ message:+ type: string+ docs: Human-readable description of the failure.+ code:+ type: PublicApiErrorCode+ docs: Stable machine-readable error code.+ details:+ type: optional<PublicApiErrorDetails>+ docs: Optional structured validation or rate-limit context.+ examples:+ - name: InvalidBody+ value:+ message: Invalid request body+ code: invalid_body+ details:+ issues:+ - code: invalid_type+ message: Invalid input+ path:+ - definition+ - name: ResourceNotFound+ value:+ message: Evaluator not found+ code: resource_not_found+ - name: RateLimited+ value:+ message: Rate limit exceeded+ code: rate_limited+ details:+ retryAfterSeconds: 60+ limit: 1000+ remaining: 0+ resetAt: "2026-03-30T10:00:00.000Z"++errors:+ BadRequestError:+ docs: Request validation failed or the requested state is invalid.+ status-code: 400+ type: PublicApiError+ UnauthorizedError:+ docs: Authentication failed.+ status-code: 401+ type: PublicApiError+ AccessDeniedError:+ docs: The caller is not allowed to access this resource.+ status-code: 403+ type: PublicApiError+ NotFoundError:+ docs: The requested evaluator or evaluation rule was not found in the authenticated project.+ status-code: 404+ type: PublicApiError+ MethodNotAllowedError:+ docs: The HTTP method is not supported by this endpoint.+ status-code: 405+ type: PublicApiError+ ConflictError:+ docs: The request conflicts with the current evaluator or evaluation-rule state.+ status-code: 409+ type: PublicApiError+ PreconditionFailedError:+ docs: The evaluator configuration cannot be used in its current form.+ status-code: 412+ type: PublicApiError+ TooManyRequestsError:+ docs: The project is rate limited. Use the response details to decide when to retry.+ status-code: 429+ type: PublicApiError+ InternalServerError:+ docs: An unexpected server-side error occurred.+ status-code: 500+ type: PublicApiErrordiff --git a/fern/apis/server/definition/evaluation-rules.yml b/fern/apis/server/definition/evaluation-rules.ymlnew file mode 100644index 0000000000..9f3853de3d--- /dev/null+++ b/fern/apis/server/definition/evaluation-rules.yml@@ -0,0 +1,241 @@+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json+imports:+ evals: ./evaluation-commons.yml+ evaluators: ./evaluators.yml+ errors: ./evaluation-errors.yml++service:+ auth: true+ base-path: /api/public/v2/evaluation-rules+ endpoints:+ create:+ docs: |+ Create an evaluation rule using stable evaluator identifiers.++ An evaluation rule defines **which** incoming observations should be evaluated and how prompt variables should be populated.++ Key behavior:+ - rule names are not identifiers and do not need to be unique+ - rules always use the latest version of each associated evaluator+ - an enabled rule requires at least one evaluator assignment+ - omit `sampling` to evaluate every matching observation+ - omit `filter` to match every incoming observation+ - `datasetId` is the public filter name for selecting experiment datasets+ method: POST+ path: ""+ request: CreateEvaluationRuleRequest+ response:+ status-code: 201+ type: EvaluationRule+ errors:+ - errors.BadRequestError+ - errors.UnauthorizedError+ - errors.AccessDeniedError+ - errors.NotFoundError+ - errors.MethodNotAllowedError+ - errors.ConflictError+ - errors.TooManyRequestsError+ - errors.InternalServerError++ list:+ docs: |+ List evaluation rules in newest-first creation order.++ This includes legacy trace and dataset rules so they can be inspected and migrated. Treat the cursor as opaque and return it unchanged.+ method: GET+ path: ""+ request:+ name: ListEvaluationRulesRequest+ query-parameters:+ limit:+ type: optional<integer>+ docs: Maximum number of items to return. Defaults to `50` and cannot exceed `100`.+ cursor:+ type: optional<string>+ docs: Opaque cursor returned by the previous page.+ response: EvaluationRulesPage+ errors:+ - errors.BadRequestError+ - errors.UnauthorizedError+ - errors.AccessDeniedError+ - errors.MethodNotAllowedError+ - errors.TooManyRequestsError+ - errors.InternalServerError++ get:+ docs: Get one evaluation rule, including a legacy trace or dataset rule, by its stable identifier.+ method: GET+ path: /{evaluationRuleId}+ path-parameters:+ evaluationRuleId:+ type: string+ docs: Stable evaluation-rule identifier returned by the evaluation-rule endpoints.+ response: EvaluationRule+ errors:+ - errors.UnauthorizedError+ - errors.AccessDeniedError+ - errors.NotFoundError+ - errors.MethodNotAllowedError+ - errors.TooManyRequestsError+ - errors.InternalServerError++ update:+ docs: |+ Update an evaluation rule by its stable identifier.++ Provide only the fields to change. Providing `evaluatorAssignments` replaces the complete assignment list. Replacing the list with an empty array disables the rule. Setting `enabled=true` is rejected when the resulting assignment list is empty, including when both fields are sent in the same request.++ Legacy trace and dataset rules follow the existing lifecycle restrictions: they can be deactivated with `enabled=false`, but their name, filters, sampling, and evaluator assignments cannot be changed.+ method: PATCH+ path: /{evaluationRuleId}+ path-parameters:+ evaluationRuleId:+ type: string+ docs: Stable evaluation-rule identifier returned by the evaluation-rule endpoints.+ request: UpdateEvaluationRuleRequest+ response: EvaluationRule+ errors:+ - errors.BadRequestError+ - errors.UnauthorizedError+ - errors.AccessDeniedError+ - errors.NotFoundError+ - errors.MethodNotAllowedError+ - errors.ConflictError+ - errors.TooManyRequestsError+ - errors.InternalServerError++ delete:+ docs: |+ Delete an evaluation rule.++ This removes the live-ingestion rule only. It does not delete associated evaluators or scores already produced by them.++ Legacy trace and dataset rules can also be deleted. Their evaluators and previously produced scores are preserved.+ method: DELETE+ path: /{evaluationRuleId}+ path-parameters:+ evaluationRuleId:+ type: string+ docs: Stable evaluation-rule identifier returned by the evaluation-rule endpoints.+ response: DeletedEvaluationRule+ errors:+ - errors.UnauthorizedError+ - errors.AccessDeniedError+ - errors.NotFoundError+ - errors.MethodNotAllowedError+ - errors.TooManyRequestsError+ - errors.InternalServerError++types:+ EvaluationRuleEvaluatorAssignmentInput:+ properties:+ evaluatorId:+ type: string+ docs: Stable evaluator identifier. The rule automatically uses that evaluator's latest version.+ variableMapping:+ type: optional<nullable<list<evals.PromptVariableMappingInput>>>+ docs: |+ Rule-specific prompt-variable mapping.++ Set to `null` or omit to inherit the evaluator's latest default mapping. Code evaluators use the fixed runtime mapping and should use `null`.++ EvaluatorAssignment:+ properties:+ evaluatorId:+ type: string+ docs: Stable identifier of the evaluator associated with this rule.+ variableMapping:+ type: nullable<list<evals.PromptVariableMapping>>+ docs: Stored rule-specific override, or `null` when the evaluator's latest default mapping is inherited. Legacy mappings are returned as an explicit `legacy` variant.++ EvaluationRule:+ docs: |+ Live evaluation rule for incoming observations.++ A rule determines which evaluators should be used, which observations should trigger scoring, how often scoring should run, and which observation fields should populate prompt variables.+ properties:+ id:+ type: string+ docs: Stable evaluation-rule identifier.+ name:+ type: string+ docs: Human-readable rule name. This is independent from evaluator names and does not need to be unique.+ createdBy:+ type: nullable<evaluators.Creator>+ docs: User who created this rule, or `null` when no user can be resolved.+ enabled:+ type: boolean+ docs: Whether live execution is enabled for this rule.+ sampling:+ type: double+ docs: |+ Fraction of matching observations that should be evaluated.++ Must be between `0` and `1`.+ - `1` evaluates every matching observation.+ - `0.25` evaluates approximately 25% of matching observations.+ filter:+ type: list<evals.EvaluationRuleReadFilter>+ docs: List of stored filter conditions returned verbatim. Filters with a `key` use the keyed response shape; all others use the base shape. These response shapes are not broken down by internal filter type. An empty list matches every incoming object.+ evaluatorAssignments:+ type: list<EvaluatorAssignment>+ docs: Evaluators attached to this rule in deterministic assignment order.+ createdAt:+ type: datetime+ docs: Timestamp when the evaluation rule was created.+ updatedAt:+ type: datetime+ docs: Timestamp when the evaluation rule was last updated.++ CreateEvaluationRuleRequest:+ properties:+ name:+ type: string+ docs: Human-readable rule name. Names are not identifiers and do not need to be unique.+ enabled:+ type: boolean+ docs: Whether live execution should start immediately. Enabled rules require at least one evaluator assignment.+ sampling:+ type: optional<double>+ docs: Fraction of matching observations to evaluate. Omit this field to use the default of `1`, which evaluates every match. `null` is not accepted.+ filter:+ type: optional<list<evals.EvaluationRuleFilter>>+ docs: Conditions used to select observations. Defaults to an empty list, which matches every incoming observation.+ evaluatorAssignments:+ type: list<EvaluationRuleEvaluatorAssignmentInput>+ docs: Evaluators to attach to this rule. Disabled rules may use an empty list as a draft.++ UpdateEvaluationRuleRequest:+ docs: At least one field must be provided.+ properties:+ name:+ type: optional<string>+ docs: New human-readable rule name.+ enabled:+ type: optional<boolean>+ docs: New desired live-execution state.+ sampling:+ type: optional<double>+ docs: New fraction of matching observations to evaluate. Omit to keep the current value.+ filter:+ type: optional<list<evals.EvaluationRuleFilter>>+ docs: Complete replacement filter list. An empty list matches every incoming observation.+ evaluatorAssignments:+ type: optional<list<EvaluationRuleEvaluatorAssignmentInput>>+ docs: Complete replacement assignment list. An empty list disables the rule.++ EvaluationRulesPage:+ properties:+ data:+ type: list<EvaluationRule>+ docs: Evaluation rules for this page.+ meta:+ type: evaluators.CursorMeta+ docs: Cursor pagination metadata.++ DeletedEvaluationRule:+ docs: Confirmation returned after successful evaluation-rule deletion.+ properties:+ id:+ type: string+ docs: Identifier of the deleted evaluation rule.diff --git a/fern/apis/server/definition/evaluators.yml b/fern/apis/server/definition/evaluators.ymlnew file mode 100644index 0000000000..45778b3fa1--- /dev/null+++ b/fern/apis/server/definition/evaluators.yml@@ -0,0 +1,529 @@+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json+imports:+ evals: ./evaluation-commons.yml+ errors: ./evaluation-errors.yml++service:+ auth: true+ base-path: /api/public/v2/evaluators+ endpoints:+ create:+ docs: |+ Create an evaluator in the authenticated project.++ An evaluator defines **how** Langfuse should score data. LLM-as-a-judge evaluators define a prompt, expected structured output, and optional model configuration. Code evaluators define source code and a runtime language.++ This always creates a new evaluator with version `1`. Names are not identifiers and do not need to be unique.+ method: POST+ path: ""+ request: CreateEvaluatorRequest+ response:+ status-code: 201+ type: Evaluator+ errors:+ - errors.BadRequestError+ - errors.UnauthorizedError+ - errors.AccessDeniedError+ - errors.MethodNotAllowedError+ - errors.PreconditionFailedError+ - errors.TooManyRequestsError+ - errors.InternalServerError++ list:+ docs: |+ List evaluators in newest-first creation order.++ Every evaluator includes its latest definition and version metadata flattened into the evaluator object, plus associated evaluation rules. Treat the cursor as opaque and return it unchanged.+ method: GET+ path: ""+ request:+ name: ListEvaluatorsRequest+ query-parameters:+ limit:+ type: optional<integer>+ docs: Maximum number of items to return. Defaults to `50` and cannot exceed `100`.+ cursor:+ type: optional<string>+ docs: Opaque cursor returned by the previous page.+ response: EvaluatorsPage+ errors:+ - errors.BadRequestError+ - errors.UnauthorizedError+ - errors.AccessDeniedError+ - errors.MethodNotAllowedError+ - errors.TooManyRequestsError+ - errors.InternalServerError++ get:+ docs: |+ Get one evaluator by its stable identifier.++ The response includes the evaluator's latest definition and version metadata flattened into the evaluator object, plus associated evaluation rules. Use the version-history endpoint when older definitions are needed.+ method: GET+ path: /{evaluatorId}+ path-parameters:+ evaluatorId:+ type: string+ docs: Stable evaluator identifier returned by the evaluator endpoints.+ response: Evaluator+ errors:+ - errors.UnauthorizedError+ - errors.AccessDeniedError+ - errors.NotFoundError+ - errors.MethodNotAllowedError+ - errors.TooManyRequestsError+ - errors.InternalServerError++ update:+ docs: |+ Update an evaluator by its stable identifier.++ Provide only the top-level fields to change. Metadata-only changes do not create a version. Evaluator type cannot change.++ Definition fields are flattened into the request. To replace a definition, include `type` and every definition field for that type. Definition fields are replaced as a complete unit rather than merged. For LLM-as-a-judge evaluators, omitting or setting `modelConfig` to `null` selects the project's default evaluation model.++ Replacing a definition automatically returns an evaluator paused by an invalid or missing model configuration to `active`, while preserving its id and evaluation-rule assignments. Pauses caused by provider authentication, billing, connectivity, account state, or an unknown legacy reason remain paused until the explicit reactivation check succeeds.+ method: PATCH+ path: /{evaluatorId}+ path-parameters:+ evaluatorId:+ type: string+ docs: Stable evaluator identifier returned by the evaluator endpoints.+ request: UpdateEvaluatorRequest+ response: Evaluator+ errors:+ - errors.BadRequestError+ - errors.UnauthorizedError+ - errors.AccessDeniedError+ - errors.NotFoundError+ - errors.MethodNotAllowedError+ - errors.ConflictError+ - errors.PreconditionFailedError+ - errors.TooManyRequestsError+ - errors.InternalServerError++ delete:+ docs: |+ Delete an evaluator and all of its stored versions.++ Associated evaluation-rule assignments are also removed. Scores already produced by the evaluator are preserved.+ method: DELETE+ path: /{evaluatorId}+ path-parameters:+ evaluatorId:+ type: string+ docs: Stable evaluator identifier returned by the evaluator endpoints.+ response: DeletedEvaluator+ errors:+ - errors.UnauthorizedError+ - errors.AccessDeniedError+ - errors.NotFoundError+ - errors.MethodNotAllowedError+ - errors.TooManyRequestsError+ - errors.InternalServerError++ listVersions:+ docs: |+ List an evaluator's version history in newest-first order.++ This endpoint is intended for history and audit use cases. Ordinary clients can use the flattened `version` and definition fields on the evaluator response.+ method: GET+ path: /{evaluatorId}/versions+ path-parameters:+ evaluatorId:+ type: string+ docs: Stable evaluator identifier returned by the evaluator endpoints.+ request:+ name: ListEvaluatorVersionsRequest+ query-parameters:+ limit:+ type: optional<integer>+ docs: Maximum number of versions to return. Defaults to `50` and cannot exceed `100`.+ cursor:+ type: optional<string>+ docs: Opaque cursor returned by the previous page.+ response: EvaluatorVersionsPage+ errors:+ - errors.BadRequestError+ - errors.UnauthorizedError+ - errors.AccessDeniedError+ - errors.NotFoundError+ - errors.MethodNotAllowedError+ - errors.TooManyRequestsError+ - errors.InternalServerError++types:+ Creator:+ docs: User who created the resource, or `null` when no user can be resolved.+ properties:+ id:+ type: string+ docs: User identifier.+ name:+ type: nullable<string>+ docs: User name, or `null` when unavailable.++ EvaluatorModelConfig:+ docs: |+ Explicit model configuration for an evaluator.++ If the complete `modelConfig` is `null`, Langfuse uses the project's default evaluation model.+ If provided, the model must be available to the project when the evaluator or evaluation rule is enabled.+ properties:+ provider:+ type: string+ docs: |+ Provider identifier, for example `openai` or `anthropic`.++ To discover valid values for the current project, call `GET /api/public/llm-connections` and use one of the returned `provider` values.+ model:+ type: string+ docs: Model identifier exposed by the provider, for example `gpt-4.1-mini`.+ examples:+ - value:+ provider: openai+ model: gpt-4.1-mini++ EvaluatorChatMessage:+ docs: One user chat message in an evaluator prompt.+ properties:+ role:+ type: literal<"user">+ content:+ type: string+ docs: Message content. Evaluator variables use `{{variable}}` syntax.++ EvaluatorChatPrompt:+ docs: A list containing exactly one user chat message.+ type: list<EvaluatorChatMessage>+ validation:+ minItems: 1+ maxItems: 1++ EvaluatorChatPromptInput:+ docs: A user prompt string, or a list containing exactly one user chat message.+ discriminated: false+ union:+ - string+ - EvaluatorChatPrompt++ EvaluatorVersionBase:+ properties:+ id:+ type: string+ docs: Stable identifier of this evaluator version.+ version:+ type: integer+ docs: Monotonically increasing evaluator version number.+ createdAt:+ type: datetime+ docs: Timestamp when this evaluator version was created.+ createdBy:+ type: nullable<Creator>+ docs: User who created this version, or `null` when no user can be resolved.++ LlmAsJudgeEvaluatorVersion:+ extends: EvaluatorVersionBase+ properties:+ type:+ type: literal<"llm_as_judge">+ docs: Evaluator type.+ prompt:+ type: EvaluatorChatPrompt+ docs: The single user chat message used during evaluation.+ variables:+ type: list<string>+ docs: |+ Variables extracted from the prompt and available for evaluation-rule mappings.++ Every variable must be mapped exactly once when a rule provides an explicit mapping.+ variableMapping:+ type: nullable<list<evals.PromptVariableMappingRead>>+ docs: Default variable mapping for this evaluator version, or `null` when no default is configured.+ modelConfig:+ type: nullable<EvaluatorModelConfig>+ docs: Explicit model configuration, or `null` when the project's default evaluation model is used.+ outputDefinition:+ type: evals.PublicEvaluatorOutputDefinition+ docs: |+ Structured output schema returned by this evaluator.++ Responses include `dataType` and omit the internal output-definition persistence version.++ CodeEvaluatorVersion:+ extends: EvaluatorVersionBase+ properties:+ type:+ type: literal<"code">+ docs: Evaluator type.+ sourceCode:+ type: string+ docs: Source code executed for each matched observation.+ sourceCodeLanguage:+ type: evals.CodeEvaluatorSourceCodeLanguage+ docs: Runtime language used to execute the source code.++ EvaluatorVersion:+ discriminant: type+ union:+ llm_as_judge:+ type: LlmAsJudgeEvaluatorVersion+ code:+ type: CodeEvaluatorVersion++ EvaluationRuleAssignment:+ docs: Evaluation-rule assignment associated with this evaluator.+ properties:+ evaluationRuleId:+ type: string+ docs: Stable identifier of the assigned evaluation rule.+ variableMappingOverride:+ type: optional<list<evals.PromptVariableMapping>>+ docs: Rule-specific variable mapping override. Omitted when the evaluator's latest default mapping is inherited. Legacy mappings use the explicit `legacy` variant.++ EvaluatorBase:+ properties:+ id:+ type: string+ docs: Stable identifier of this evaluator across all versions.+ name:+ type: string+ docs: Human-readable evaluator name. Names are not identifiers and do not need to be unique.+ description:+ type: nullable<string>+ docs: Optional human-readable evaluator description.+ createdBy:+ type: nullable<Creator>+ docs: User who created this evaluator, or `null` when no user can be resolved.+ status:+ type: EvaluatorStatus+ docs: Effective evaluator status after Langfuse validates its runtime configuration.+ pausedAt:+ type: nullable<datetime>+ docs: Timestamp when the evaluator was paused, otherwise `null`.+ pausedReason:+ type: nullable<string>+ docs: Machine-readable reason when `status=paused`, otherwise `null`.+ pausedMessage:+ type: nullable<string>+ docs: Human-readable explanation when `status=paused`, otherwise `null`.+ evaluationRuleAssignments:+ type: list<EvaluationRuleAssignment>+ docs: All modern and legacy evaluation-rule assignments in newest-assignment-first order. Rule-specific mappings are exposed as `variableMappingOverride`; inherited defaults are omitted.+ createdAt:+ type: datetime+ docs: Timestamp when the evaluator was created.+ updatedAt:+ type: datetime+ docs: Timestamp when the evaluator was last updated.+ versionId:+ type: string+ docs: Stable identifier of the latest evaluator version.+ version:+ type: integer+ docs: Monotonically increasing latest evaluator version number.+ versionCreatedAt:+ type: datetime+ docs: Timestamp when the latest evaluator version was created.+ versionCreatedBy:+ type: nullable<Creator>+ docs: User who created the latest evaluator version, or `null` when no user can be resolved.++ LlmAsJudgeEvaluator:+ extends: EvaluatorBase+ properties:+ type:+ type: literal<"llm_as_judge">+ docs: Evaluator type.+ prompt:+ type: EvaluatorChatPrompt+ docs: The single user chat message used by the latest evaluator version.+ variables:+ type: list<string>+ docs: Variables extracted from the latest prompt and available for evaluation-rule mappings.+ variableMapping:+ type: nullable<list<evals.PromptVariableMappingRead>>+ docs: Default variable mapping for the latest version, or `null` when no default is configured.+ modelConfig:+ type: nullable<EvaluatorModelConfig>+ docs: Explicit model configuration for the latest version, or `null` when the project's default evaluation model is used.+ outputDefinition:+ type: evals.PublicEvaluatorOutputDefinition+ docs: Structured output schema returned by the latest evaluator version.++ CodeEvaluator:+ extends: EvaluatorBase+ properties:+ type:+ type: literal<"code">+ docs: Evaluator type.+ sourceCode:+ type: string+ docs: Source code executed by the latest evaluator version.+ sourceCodeLanguage:+ type: evals.CodeEvaluatorSourceCodeLanguage+ docs: Runtime language used to execute the latest source code.++ Evaluator:+ docs: |+ One evaluator that can be used for scoring.++ An evaluator describes **how** to score data. Associated evaluation rules describe **which** live objects should be evaluated.+ discriminant: type+ union:+ llm_as_judge:+ type: LlmAsJudgeEvaluator+ code:+ type: CodeEvaluator++ EvaluatorStatus:+ docs: |+ Effective evaluator runtime status.++ - `active`: the evaluator can run.+ - `paused`: Langfuse paused execution until the underlying issue is resolved.+ enum:+ - active+ - paused++ CreateLlmAsJudgeEvaluatorRequest:+ properties:+ name:+ type: string+ docs: Human-readable evaluator name. Names are not identifiers and do not need to be unique.+ description:+ type: optional<nullable<string>>+ docs: Optional human-readable evaluator description.+ type:+ type: literal<"llm_as_judge">+ docs: Evaluator type.+ prompt:+ type: EvaluatorChatPromptInput+ docs: User prompt string shortcut or a list containing exactly one user chat message.+ modelConfig:+ type: optional<nullable<EvaluatorModelConfig>>+ docs: Explicit model configuration. Set to `null` or omit to use the project's default evaluation model.+ variableMapping:+ type: optional<nullable<list<evals.PromptVariableMappingInput>>>+ docs: Default prompt-variable mapping, or `null` when no default is configured.+ outputDefinition:+ type: evals.EvaluatorOutputDefinition+ docs: Structured output schema returned by this evaluator. Do not include an internal persistence version.++ CreateCodeEvaluatorRequest:+ properties:+ name:+ type: string+ docs: Human-readable evaluator name. Names are not identifiers and do not need to be unique.+ description:+ type: optional<nullable<string>>+ docs: Optional human-readable evaluator description.+ type:+ type: literal<"code">+ docs: Evaluator type.+ sourceCode:+ type: string+ docs: Source code executed for each matched observation.+ sourceCodeLanguage:+ type: evals.CodeEvaluatorSourceCodeLanguage+ docs: Runtime language used to execute the source code.++ CreateEvaluatorRequest:+ docs: Initial evaluator definition with metadata and definition fields at the same level. The returned evaluator starts at version `1`.+ discriminant: type+ union:+ llm_as_judge:+ type: CreateLlmAsJudgeEvaluatorRequest+ code:+ type: CreateCodeEvaluatorRequest++ UpdateEvaluatorMetadataRequest:+ docs: Metadata-only update. At least one field must be provided.+ properties:+ name:+ type: optional<string>+ docs: New human-readable evaluator name.+ description:+ type: optional<nullable<string>>+ docs: New description. Set to `null` to clear it.++ UpdateLlmAsJudgeEvaluatorRequest:+ properties:+ name:+ type: optional<string>+ docs: New human-readable evaluator name.+ description:+ type: optional<nullable<string>>+ docs: New description. Set to `null` to clear it.+ type:+ type: literal<"llm_as_judge">+ docs: Evaluator type. The type of an existing evaluator cannot change.+ prompt:+ type: EvaluatorChatPromptInput+ docs: Complete replacement user prompt string or a list containing exactly one user chat message.+ modelConfig:+ type: optional<nullable<EvaluatorModelConfig>>+ docs: Explicit model configuration. Set to `null` or omit to use the project's default evaluation model.+ variableMapping:+ type: optional<nullable<list<evals.PromptVariableMappingInput>>>+ docs: Complete replacement default variable mapping, or `null` when no default is configured.+ outputDefinition:+ type: evals.EvaluatorOutputDefinition+ docs: Complete replacement output schema. Do not include an internal persistence version.++ UpdateCodeEvaluatorRequest:+ properties:+ name:+ type: optional<string>+ docs: New human-readable evaluator name.+ description:+ type: optional<nullable<string>>+ docs: New description. Set to `null` to clear it.+ type:+ type: literal<"code">+ docs: Evaluator type. The type of an existing evaluator cannot change.+ sourceCode:+ type: string+ docs: Complete replacement source code.+ sourceCodeLanguage:+ type: evals.CodeEvaluatorSourceCodeLanguage+ docs: Runtime language used to execute the source code.++ UpdateEvaluatorRequest:+ docs: Metadata-only update or complete flattened definition replacement. At least one field must be provided.+ discriminated: false+ union:+ - UpdateEvaluatorMetadataRequest+ - UpdateLlmAsJudgeEvaluatorRequest+ - UpdateCodeEvaluatorRequest++ EvaluatorsPage:+ properties:+ data:+ type: list<Evaluator>+ docs: Evaluators for this page.+ meta:+ type: CursorMeta+ docs: Cursor pagination metadata.++ EvaluatorVersionsPage:+ properties:+ data:+ type: list<EvaluatorVersion>+ docs: Evaluator versions for this page in newest-first order.+ meta:+ type: CursorMeta+ docs: Cursor pagination metadata.++ CursorMeta:+ properties:+ cursor:+ type: optional<string>+ docs: Opaque cursor for the next page. Omitted when there is no next page.++ DeletedEvaluator:+ docs: Confirmation returned after successful evaluator deletion.+ properties:+ id:+ type: string+ docs: Identifier of the deleted evaluator.diff --git a/fern/apis/server/definition/unstable/commons.yml b/fern/apis/server/definition/unstable/commons.ymlindex b616d1435f..48e0c0a72a 100644--- a/fern/apis/server/definition/unstable/commons.yml+++ b/fern/apis/server/definition/unstable/commons.yml@@ -2,7 +2,7 @@types:EvaluatorType:docs: |- The evaluator engine type.+ The evaluator type.The unstable public API supports LLM-as-a-judge and code evaluators.enum:@@ -41,7 +41,7 @@ types:- inactive- paused- EvaluationRuleMappingSource:+ PromptVariableMappingSource:docs: |Source field used to populate a prompt variable.@@ -422,9 +422,9 @@ types:type: optional<string>docs: Ignored placeholder value. Clients may omit it or send an empty string.- EvaluationRuleMapping:+ PromptVariableMappingInput:docs: |- Maps one evaluator variable to one source field from the target object.+ Connects one prompt variable to the available evaluation data.Manual mappings are used for `llm_as_judge` evaluators. `code` evaluators use a fixed runtime mapping managed by Langfuse.@@ -439,8 +439,8 @@ types:Recovery guidance:- `invalid_variable_mapping`: the variable name is unknown for this evaluator, or the selected `source` is not valid for the chosen `target`- - `missing_variable_mapping`: one or more LLM-as-judge evaluator variables are not mapped yet- - `duplicate_variable_mapping`: the same evaluator variable appears more than once+ - `missing_variable_mapping`: one or more LLM-as-judge prompt variables are not mapped yet+ - `duplicate_variable_mapping`: the same prompt variable appears more than once- `invalid_json_path`: the JSONPath expression is malformed. Remove it or correct it.properties:variable:@@ -450,7 +450,7 @@ types:Example: for the prompt `Judge {{input}} against {{output}}`, use `input` and `output`.source:- type: EvaluationRuleMappingSource+ type: PromptVariableMappingSourcedocs: |Source field that should populate the prompt variable.@@ -481,23 +481,17 @@ types:variable: expected_outputsource: expected_output- EvaluationRuleReadMapping:+ PromptVariableMappingRead:docs: |- Maps an evaluator variable to target data, as returned by read endpoints.+ Connects one prompt variable to data from a target.- Reads are more permissive than writes in two ways, so that data configured before a- migration stays readable and repairable:- - `source` is `null` when a mapping was never fully configured. Set a concrete `source` to fix it.- - every `source` value is allowed regardless of the rule `target`, because an evaluator's- default mapping is stored independently of the rules that use it.-- Requests always require a concrete `source` that is valid for the selected `target`.+ `source` is `null` when the mapping is incomplete.properties:variable:type: stringdocs: Prompt variable name without braces.source:- type: nullable<EvaluationRuleMappingSource>+ type: nullable<PromptVariableMappingSource>docs: Source field populating the variable, or `null` when the mapping is incomplete.jsonPath:type: optional<string>diff --git a/fern/apis/server/definition/unstable/evaluation-rules.yml b/fern/apis/server/definition/unstable/evaluation-rules.ymlindex 5de05f3a9a..ccf0d16187 100644--- a/fern/apis/server/definition/unstable/evaluation-rules.yml+++ b/fern/apis/server/definition/unstable/evaluation-rules.yml@@ -9,6 +9,9 @@ service:base-path: /api/public/unstableendpoints:create:+ availability:+ status: deprecated+ message: "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4."docs: |Create an evaluation rule.@@ -236,6 +239,9 @@ service:updatedAt: "2026-03-30T09:30:00.000Z"list:+ availability:+ status: deprecated+ message: "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4."docs: |List evaluation rules in the authenticated project.@@ -261,6 +267,9 @@ service:- errors.InternalServerErrorget:+ availability:+ status: deprecated+ message: "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4."docs: |Get one evaluation rule by its identifier.@@ -282,6 +291,9 @@ service:- errors.InternalServerErrorupdate:+ availability:+ status: deprecated+ message: "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4."docs: |Update an evaluation rule.@@ -323,6 +335,9 @@ service:- errors.InternalServerErrordelete:+ availability:+ status: deprecated+ message: "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4."docs: |Delete an evaluation rule.@@ -352,7 +367,7 @@ types:- which evaluator should be used- which target objects should trigger scoring- how often scoring should run- - which target fields should populate each evaluator variable+ - which target fields should populate each prompt variable- whether the deployment is active, inactive, or pausedImportant status semantics:@@ -414,7 +429,7 @@ types:type: list<commons.EvaluationRuleFilter>docs: List of filter conditions used to decide whether a target should be evaluated.mapping:- type: list<commons.EvaluationRuleReadMapping>+ type: list<commons.PromptVariableMappingRead>docs: Deprecated compatibility alias containing the effective mapping for `evaluators[0]`.examples:- value:@@ -456,9 +471,9 @@ types:LegacyEvaluationRule:docs: |- Legacy trace- or dataset-level evaluation rule returned by list and get for migration.+ **Deprecated:** Legacy trace- or dataset-level evaluation rule returned by list and get for migration.- This resource is read-only through the unstable public API. Its mapping preserves the trace, dataset item, or named observation selected for each evaluator variable.+ This resource is read-only through the unstable public API. Its mapping preserves the trace, dataset item, or named observation selected for each prompt variable.extends: EvaluationRuleBaseproperties:evaluators:@@ -476,7 +491,7 @@ types:type: list<commons.EvaluationRuleFilter>docs: Stored filters used by the legacy trace or dataset rule.mapping:- type: list<LegacyEvaluationRuleMapping>+ type: list<LegacyPromptVariableMapping>docs: Stored variable mappings, including the trace, dataset item, or named observation selected for each variable.ReadableEvaluationRule:@@ -504,7 +519,7 @@ types:- reference an existing evaluator family by `evaluator.name` and `evaluator.type`- choose `target=observation` or `target=experiment`- if `target=experiment` and you want a dataset filter, call `GET /api/public/v2/datasets` first and use dataset `id` values in `filter[].value`- - for `llm_as_judge`, fetch or inspect the evaluator first and provide a complete variable mapping for every evaluator variable+ - for `llm_as_judge`, fetch or inspect the evaluator first and provide a complete mapping for every prompt variable- for `code`, do not send variables or mappings; Langfuse stores the fixed code runtime mapping automatically- optionally narrow execution with `filter`- set `enabled=true` only when you want live execution immediately@@ -564,12 +579,12 @@ types:Each filter object must use a column that is valid for that `target`.For `target=experiment`, `column=datasetId` expects dataset `id` values from `GET /api/public/v2/datasets`, not dataset names.mapping:- type: list<commons.EvaluationRuleMapping>+ type: list<commons.PromptVariableMappingInput>docs: |LLM-as-judge variable mappings.- Every evaluator variable must appear exactly once.- Build this list from the evaluator `variables` array returned by the evaluator endpoints.+ Every prompt variable must appear exactly once.+ Build this list from the evaluator's `variables` array.CreateCodeEvaluationRuleRequest:properties:@@ -612,7 +627,7 @@ types:- If you only want to rename the rule or change sampling, send just those fields.- To add, remove, or remap evaluators, send `evaluators`. It replaces the whole assignment set, so include every evaluator the rule should keep.- `evaluators` cannot be combined with the deprecated `evaluator`/`mapping` pair, which only ever addressed the first assignment.- - If you change to an LLM-as-judge `evaluator`, send a fresh `mapping` unless you are certain the existing mapping still matches the evaluator variables.+ - If you change to an LLM-as-judge `evaluator`, send a fresh `mapping` unless you are certain the existing mapping still matches the prompt variables.- If you change `target` for an LLM-as-judge rule, usually send both `filter` and `mapping` in the same request.- For code evaluator rules, omit `mapping`; Langfuse stores the fixed code runtime mapping automatically.- If you change an experiment `datasetId` filter, call `GET /api/public/v2/datasets` and use dataset `id` values from that response.@@ -650,7 +665,7 @@ types:For `target=experiment`, `column=datasetId` expects dataset `id` values from `GET /api/public/v2/datasets`, not dataset names.mapping:- type: optional<list<commons.EvaluationRuleMapping>>+ type: optional<list<commons.PromptVariableMappingInput>>docs: |Updated LLM-as-judge variable mappings.@@ -682,7 +697,7 @@ types:docs: Evaluator family name.type:type: optional<LlmAsJudgeEvaluatorType>- docs: Evaluator engine type. Defaults to `llm_as_judge` when omitted.+ docs: Evaluator type. Defaults to `llm_as_judge` when omitted.CodeEvaluationRuleEvaluatorReference:docs: Code evaluator family reference used when creating an evaluation rule.@@ -713,22 +728,23 @@ types:docs: Evaluator family name.type:type: commons.EvaluatorType- docs: Evaluator engine type.+ docs: Evaluator type.EvaluationRuleEvaluatorAssignment:properties:evaluator:type: EvaluationRuleEvaluatormapping:- type: nullable<list<commons.EvaluationRuleReadMapping>>+ type: nullable<list<commons.PromptVariableMappingRead>>docs: Rule-specific override, or `null` to inherit the evaluator version's default mapping.LegacyEvaluationRuleEvaluatorAssignment:+ docs: "**Deprecated:** Evaluator assignment on a legacy trace or dataset rule."properties:evaluator:type: EvaluationRuleEvaluatormapping:- type: nullable<list<LegacyEvaluationRuleMapping>>+ type: nullable<list<LegacyPromptVariableMapping>>EvaluationRuleTimeScope:enum:@@ -736,12 +752,13 @@ types:- EXISTINGLegacyEvaluationRuleTarget:+ docs: "**Deprecated:** Legacy trace or dataset evaluation target."enum:- trace- dataset- LegacyEvaluationRuleMapping:- docs: Maps one evaluator variable to a trace, dataset item, or field on a named observation in a legacy rule.+ LegacyPromptVariableMapping:+ docs: "**Deprecated:** Connects one prompt variable to data from a trace, dataset item, or named observation in a legacy rule."properties:variable:type: string@@ -756,6 +773,7 @@ types:type: optional<string>LegacyEvaluationObject:+ docs: "**Deprecated:** Legacy Langfuse object used by a trace or dataset evaluation rule."enum:- trace- span@@ -775,7 +793,7 @@ types:evaluator:type: CreateEvaluationRuleEvaluatorReferencemapping:- type: optional<list<commons.EvaluationRuleMapping>>+ type: optional<list<commons.PromptVariableMappingInput>>docs: Optional rule-specific override. Omit to inherit the evaluator version's default mapping. Omit for code evaluators.CreateEvaluationRuleEvaluatorReference:diff --git a/fern/apis/server/definition/unstable/evaluators.yml b/fern/apis/server/definition/unstable/evaluators.ymlindex e5a6eee91a..622adf79c1 100644--- a/fern/apis/server/definition/unstable/evaluators.yml+++ b/fern/apis/server/definition/unstable/evaluators.yml@@ -9,6 +9,9 @@ service:base-path: /api/public/unstableendpoints:create:+ availability:+ status: deprecated+ message: "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluators` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4."docs: |Create an evaluator in the authenticated project.@@ -257,6 +260,9 @@ service:updatedAt: "2026-03-30T09:15:00.000Z"list:+ availability:+ status: deprecated+ message: "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluators` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4."docs: |List the evaluators available to the authenticated project.@@ -284,6 +290,9 @@ service:- errors.InternalServerErrorget:+ availability:+ status: deprecated+ message: "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluators` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4."docs: |Get one evaluator by `id`.@@ -305,6 +314,9 @@ service:- errors.InternalServerErrordelete:+ availability:+ status: deprecated+ message: "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluators` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4."docs: |Delete an evaluator.@@ -378,7 +390,7 @@ types:LLM evaluators require every variable to be mapped exactly once. Code evaluators always expose the fixed runtime payload fields and Langfuse maps them automatically.mapping:- type: nullable<list<commons.EvaluationRuleReadMapping>>+ type: nullable<list<commons.PromptVariableMappingRead>>docs: |Default variable mapping for this evaluator version, or `null` when no default is configured.@@ -462,7 +474,7 @@ types:type: optional<nullable<commons.EvaluatorModelConfig>>docs: Optional explicit model configuration. Omit or set to `null` to use the project default evaluation model.mapping:- type: optional<list<commons.EvaluationRuleMapping>>+ type: optional<list<commons.PromptVariableMappingInput>>docs: Optional default variable mapping inherited by rule assignments that do not provide an override.CreateCodeEvaluatorRequest:diff --git a/packages/shared/src/features/evals/codeEvalConstants.ts b/packages/shared/src/features/evals/codeEvalConstants.tsnew file mode 100644index 0000000000..7b920c1025--- /dev/null+++ b/packages/shared/src/features/evals/codeEvalConstants.ts@@ -0,0 +1 @@+export const CODE_EVAL_SOURCE_MAX_BYTES = 256 * 1024;diff --git a/packages/shared/src/features/evals/evalConfigBlocking.ts b/packages/shared/src/features/evals/evalConfigBlocking.tsindex b6db3165d0..bb599ff227 100644--- a/packages/shared/src/features/evals/evalConfigBlocking.ts+++ b/packages/shared/src/features/evals/evalConfigBlocking.ts@@ -112,6 +112,22 @@ export function getEvaluatorBlockMetadata(return EVALUATOR_BLOCK_METADATA[reason];}+const DEFINITION_UPDATE_RECOVERABLE_BLOCK_REASONS: ReadonlySet<EvaluatorBlockReason> =+ new Set([+ EvaluatorBlockReason.LLM_CONNECTION_MISSING,+ EvaluatorBlockReason.DEFAULT_EVAL_MODEL_MISSING,+ EvaluatorBlockReason.EVAL_MODEL_CONFIG_INVALID,+ EvaluatorBlockReason.EVAL_MODEL_UNAVAILABLE,+ ]);++export function isEvaluatorBlockReasonRecoverableByDefinitionUpdate(+ reason: EvaluatorBlockReason | null | undefined,+) {+ return reason+ ? DEFINITION_UPDATE_RECOVERABLE_BLOCK_REASONS.has(reason)+ : false;+}+export function getEvaluatorBlockResolutionPath(params: {projectId: string;blockReason: EvaluatorBlockReason;diff --git a/packages/shared/src/features/evals/outputDefinition.test.ts b/packages/shared/src/features/evals/outputDefinition.test.tsnew file mode 100644index 0000000000..f17701309a--- /dev/null+++ b/packages/shared/src/features/evals/outputDefinition.test.ts@@ -0,0 +1,63 @@+import { describe, expect, it } from "vitest";+import {+ EvalOutputDefinitionSchema,+ PersistedEvalOutputDefinitionSchema,+} from "./outputDefinition";++describe("evaluator output definitions", () => {+ it("uses the same versionless shape in the service and persistence", () => {+ const definition = EvalOutputDefinitionSchema.parse({+ dataType: "BOOLEAN",+ reasoning: { description: "Why the verdict was selected" },+ score: { description: "Whether the input passes" },+ });++ expect(PersistedEvalOutputDefinitionSchema.parse(definition)).toEqual(+ definition,+ );+ expect(definition).not.toHaveProperty("version");+ });++ it("defaults omitted score descriptions", () => {+ expect(+ EvalOutputDefinitionSchema.parse({+ dataType: "NUMERIC",+ reasoning: {},+ score: { minValue: 0, maxValue: 1 },+ }),+ ).toEqual({+ dataType: "NUMERIC",+ reasoning: { description: "" },+ score: { description: "", minValue: 0, maxValue: 1 },+ });+ });++ it("continues to read existing definitions that contain a version", () => {+ expect(+ PersistedEvalOutputDefinitionSchema.parse({+ version: 2,+ dataType: "BOOLEAN",+ reasoning: { description: "Why the verdict was selected" },+ score: { description: "Whether the input passes" },+ }),+ ).toEqual({+ dataType: "BOOLEAN",+ reasoning: { description: "Why the verdict was selected" },+ score: { description: "Whether the input passes" },+ });+ });++ it("applies categorical validation before persistence", () => {+ expect(+ EvalOutputDefinitionSchema.safeParse({+ dataType: "CATEGORICAL",+ reasoning: { description: "Why the category was selected" },+ score: {+ description: "Selected category",+ categories: ["duplicate", "duplicate"],+ shouldAllowMultipleMatches: false,+ },+ }).success,+ ).toBe(false);+ });+});diff --git a/packages/shared/src/features/evals/outputDefinition.ts b/packages/shared/src/features/evals/outputDefinition.tsindex ae64fcb547..6d03c5d5d3 100644--- a/packages/shared/src/features/evals/outputDefinition.ts+++ b/packages/shared/src/features/evals/outputDefinition.ts@@ -19,7 +19,7 @@ export type LegacyEvalOutputDefinition = z.infer<>;const EvalOutputFieldDefinitionSchema = z.object({- description: z.string().trim().min(1),+ description: z.string().trim().default(""),});export const MinimumCategoricalCategoryCount = 2;@@ -81,68 +81,66 @@ const NumericEvalOutputScoreDefinitionSchema =},);-export const NumericEvalOutputDefinitionV2Schema = z.object({- version: z.literal(2),+const NumericEvalOutputDefinitionSchema = z.object({dataType: z.literal(ScoreDataTypeEnum.NUMERIC),reasoning: EvalOutputFieldDefinitionSchema,score: NumericEvalOutputScoreDefinitionSchema,});-export type NumericEvalOutputDefinitionV2 = z.infer<- typeof NumericEvalOutputDefinitionV2Schema->;-export const BooleanEvalOutputDefinitionV2Schema = z.object({- version: z.literal(2),+const BooleanEvalOutputDefinitionSchema = z.object({dataType: z.literal(ScoreDataTypeEnum.BOOLEAN),reasoning: EvalOutputFieldDefinitionSchema,score: EvalOutputFieldDefinitionSchema,});-export type BooleanEvalOutputDefinitionV2 = z.infer<- typeof BooleanEvalOutputDefinitionV2Schema->;-export const CategoricalEvalOutputDefinitionV2Schema = z+function validateCategoricalOutputDefinition(+ value: { score: { categories: string[] } },+ ctx: z.RefinementCtx,+) {+ getCategoricalCategoryRuleViolations(value.score.categories).forEach(+ (violation) => {+ switch (violation.type) {+ case "minimum_count":+ ctx.addIssue({+ code: "custom",+ message: getMinimumCategoricalCategoriesMessage(),+ path: ["score", "categories"],+ });+ return;+ case "duplicate_value":+ ctx.addIssue({+ code: "custom",+ message: "Categories must be unique",+ path: ["score", "categories", violation.index],+ });+ return;+ }+ },+ );+}++const CategoricalEvalOutputDefinitionSchema = z.object({- version: z.literal(2),dataType: z.literal(ScoreDataTypeEnum.CATEGORICAL),reasoning: EvalOutputFieldDefinitionSchema,score: z.object({- description: z.string().trim().min(1),+ description: z.string().trim().default(""),categories: z.array(EvalCategoricalCategorySchema),shouldAllowMultipleMatches: z.boolean().default(false),}),})- .superRefine((value, ctx) => {- getCategoricalCategoryRuleViolations(value.score.categories).forEach(- (violation) => {- switch (violation.type) {- case "minimum_count":- ctx.addIssue({- code: "custom",- message: getMinimumCategoricalCategoriesMessage(),- path: ["score", "categories"],- });- return;- case "duplicate_value":- ctx.addIssue({- code: "custom",- message: "Categories must be unique",- path: ["score", "categories", violation.index],- });- return;- }- },- );- });-export type CategoricalEvalOutputDefinitionV2 = z.infer<- typeof CategoricalEvalOutputDefinitionV2Schema->;+ .superRefine(validateCategoricalOutputDefinition);++export const EvalOutputDefinitionSchema = z.union([+ NumericEvalOutputDefinitionSchema,+ BooleanEvalOutputDefinitionSchema,+ CategoricalEvalOutputDefinitionSchema,+]);+export type EvalOutputDefinition = z.infer<typeof EvalOutputDefinitionSchema>;export const PersistedEvalOutputDefinitionSchema = z.union([LegacyEvalOutputDefinitionSchema,- NumericEvalOutputDefinitionV2Schema,- BooleanEvalOutputDefinitionV2Schema,- CategoricalEvalOutputDefinitionV2Schema,+ EvalOutputDefinitionSchema,]);export type PersistedEvalOutputDefinition = z.infer<typeof PersistedEvalOutputDefinitionSchema@@ -196,7 +194,7 @@ export type EvalOutputResult =export function resolvePersistedEvalOutputDefinition(outputDefinition: PersistedEvalOutputDefinition,): ResolvedEvalOutputDefinition {- if (!("version" in outputDefinition)) {+ if (!("dataType" in outputDefinition)) {return {dataType: ScoreDataTypeEnum.NUMERIC,reasoningDescription: outputDefinition.reasoning,@@ -238,8 +236,7 @@ export function createNumericEvalOutputDefinition(params: {minValue?: number;maxValue?: number;}) {- return NumericEvalOutputDefinitionV2Schema.parse({- version: 2,+ return NumericEvalOutputDefinitionSchema.parse({dataType: ScoreDataTypeEnum.NUMERIC,reasoning: {description: params.reasoningDescription,@@ -256,8 +253,7 @@ export function createBooleanEvalOutputDefinition(params: {reasoningDescription: string;scoreDescription: string;}) {- return BooleanEvalOutputDefinitionV2Schema.parse({- version: 2,+ return BooleanEvalOutputDefinitionSchema.parse({dataType: ScoreDataTypeEnum.BOOLEAN,reasoning: {description: params.reasoningDescription,@@ -274,8 +270,7 @@ export function createCategoricalEvalOutputDefinition(params: {categories: string[];shouldAllowMultipleMatches?: boolean;}) {- return CategoricalEvalOutputDefinitionV2Schema.parse({- version: 2,+ return CategoricalEvalOutputDefinitionSchema.parse({dataType: ScoreDataTypeEnum.CATEGORICAL,reasoning: {description: params.reasoningDescription,diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.tsindex 79a61a1b02..db37c15a3d 100644--- a/packages/shared/src/index.ts+++ b/packages/shared/src/index.ts@@ -39,6 +39,7 @@ export * from "./server/llm/promptToolConfig";export * from "./features/evals/types";export * from "./features/evals/evalExecutionMetadata";export * from "./features/evals/outputDefinition";+export * from "./features/evals/codeEvalConstants";export {extractValueFromObject,extractValueFromObjectAsString,diff --git a/packages/shared/src/server/evals/codeEvalDispatcherTypes.ts b/packages/shared/src/server/evals/codeEvalDispatcherTypes.tsindex bec59d4324..8581bd6e1e 100644--- a/packages/shared/src/server/evals/codeEvalDispatcherTypes.ts+++ b/packages/shared/src/server/evals/codeEvalDispatcherTypes.ts@@ -4,8 +4,8 @@ import type {CodeEvalTemplateVariable,ToolCallForEval,} from "../../features/evals/observationForEval";--export const CODE_EVAL_SOURCE_MAX_BYTES = 256 * 1024;+export { CODE_EVAL_SOURCE_MAX_BYTES } from "../../features/evals/codeEvalConstants";+import { CODE_EVAL_SOURCE_MAX_BYTES } from "../../features/evals/codeEvalConstants";export const CODE_EVAL_DISPATCH_PAYLOAD_MAX_BYTES = 5.5 * 1024 * 1024;export const CODE_EVAL_DISPATCH_RESULT_MAX_BYTES = 256 * 1024;export const CODE_EVAL_DOCS_URL =diff --git a/web/public/generated/api/openapi.yml b/web/public/generated/api/openapi.ymlindex e6435ad853..a2008c9afa 100644--- a/web/public/generated/api/openapi.yml+++ b/web/public/generated/api/openapi.yml@@ -1597,99 +1597,41 @@ paths:schema: {}security: *ref_0deprecated: true- /api/public/experiments:- get:- description: |-- List experiments with cursor-based pagination. Results are ordered by- latest experiment activity descending.- operationId: experiments_list+ /api/public/v2/evaluation-rules:+ post:+ description: >-+ Create an evaluation rule using stable evaluator identifiers.+++ An evaluation rule defines **which** incoming observations should be+ evaluated and how prompt variables should be populated.+++ Key behavior:++ - rule names are not identifiers and do not need to be unique++ - rules always use the latest version of each associated evaluator++ - an enabled rule requires at least one evaluator assignment++ - omit `sampling` to evaluate every matching observation++ - omit `filter` to match every incoming observation++ - `datasetId` is the public filter name for selecting experiment+ datasets+ operationId: evaluationRules_createtags:- - Experiments- parameters:- - name: fields- in: query- description: |-- Comma-separated list of field groups to include. Available groups:- `core`, `metadata`, `scores`. If omitted, `core` is returned.- required: false- schema:- type: string- nullable: true- - name: limit- in: query- description: Number of experiments to return per page. Maximum 100, default 50.- required: false- schema:- type: integer- nullable: true- - name: scoreLimit- in: query- description: >-- Number of scores to return per experiment when `fields=scores` is- requested. Maximum 50, default 50.- required: false- schema:- type: integer- nullable: true- - name: cursor- in: query- description: Versioned base64url cursor from the previous response page.- required: false- schema:- type: string- nullable: true- - name: fromStartTime- in: query- description: Retrieve only experiments on or after this datetime.- required: true- schema:- type: string- format: date-time- - name: toStartTime- in: query- description: Retrieve only experiments before this datetime.- required: false- schema:- type: string- format: date-time- nullable: true- - name: id- in: query- description: Comma-separated list of experiment IDs.- required: false- schema:- type: string- nullable: true- - name: name- in: query- description: Comma-separated list of experiment names.- required: false- schema:- type: string- nullable: true- - name: datasetId- in: query- description: Comma-separated list of dataset IDs.- required: false- schema:- type: string- nullable: true- - name: filter- in: query- description: |-- JSON string containing an array of structured filter conditions.- Supported columns are `id`, `name`, and `datasetId`.- required: false- schema:- type: string- nullable: true+ - EvaluationRules+ parameters: []responses:- '200':+ '201':description: ''content:application/json:schema:- $ref: '#/components/schemas/ExperimentsResponse'+ $ref: '#/components/schemas/EvaluationRule''400':description: ''content:@@ -1715,103 +1657,54 @@ paths:content:application/json:schema: {}+ '409':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/PublicApiError'+ '429':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/PublicApiError'+ '500':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/PublicApiError'security: *ref_0- /api/public/experiment-items:+ requestBody:+ required: true+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/CreateEvaluationRuleRequest'get:- description: |-- List experiment items with cursor-based pagination. Use this endpoint- to export experiment item inputs, outputs, expected outputs, metadata,- and optionally item/trace scores. Results are ordered by time- descending.- operationId: experiments_listItems+ description: >-+ List evaluation rules in newest-first creation order.+++ This includes legacy trace and dataset rules so they can be inspected+ and migrated. Treat the cursor as opaque and return it unchanged.+ operationId: evaluationRules_listtags:- - Experiments+ - EvaluationRulesparameters:- - name: fields- in: query- description: |-- Comma-separated list of field groups to include. Available groups:- `core`, `dataset`, `io`, `metadata`, `itemMetadata`,- `experimentMetadata`, `scores`. If omitted, `core,dataset` is- returned.- required: false- schema:- type: string- nullable: true- name: limitin: querydescription: >-- Number of experiment items to return per page. Maximum 100, default- 50.- required: false- schema:- type: integer- nullable: true- - name: scoreLimit- in: query- description: >-- Number of scores to return per experiment item when `fields=scores`- is requested. Maximum 50, default 50.+ Maximum number of items to return. Defaults to `50` and cannot+ exceed `100`.required: falseschema:type: integernullable: true- name: cursorin: query- description: Versioned base64url cursor from the previous response page.- required: false- schema:- type: string- nullable: true- - name: fromStartTime- in: query- description: Retrieve only experiment items started on or after this datetime.- required: true- schema:- type: string- format: date-time- - name: toStartTime- in: query- description: Retrieve only experiment items started before this datetime.- required: false- schema:- type: string- format: date-time- nullable: true- - name: experimentId- in: query- description: Comma-separated list of experiment IDs.- required: false- schema:- type: string- nullable: true- - name: experimentName- in: query- description: Comma-separated list of experiment names.- required: false- schema:- type: string- nullable: true- - name: experimentItemId- in: query- description: Comma-separated list of experiment item IDs.- required: false- schema:- type: string- nullable: true- - name: datasetId- in: query- description: Comma-separated list of dataset IDs.- required: false- schema:- type: string- nullable: true- - name: filter- in: query- description: |-- JSON string containing an array of structured filter conditions.- Supported columns are `experimentId`, `experimentName`,- `experimentItemId`, and `datasetId`.+ description: Opaque cursor returned by the previous page.required: falseschema:type: string@@ -1822,7 +1715,7 @@ paths:content:application/json:schema:- $ref: '#/components/schemas/ExperimentItemsResponse'+ $ref: '#/components/schemas/EvaluationRulesPage''400':description: ''content:@@ -1848,27 +1741,43 @@ paths:content:application/json:schema: {}+ '429':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/PublicApiError'+ '500':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/PublicApiError'security: *ref_0- /api/public/feedback:- post:+ /api/public/v2/evaluation-rules/{evaluationRuleId}:+ get:description: >-- Submit explicit user-approved feedback about Langfuse skills, MCP tools,- CLI, docs, or public API. Do not include secrets, credentials, customer- data, trace payloads, or unrelated use-case details.--- Not available in the HIPAA cloud region.- operationId: feedback_submit+ Get one evaluation rule, including a legacy trace or dataset rule, by+ its stable identifier.+ operationId: evaluationRules_gettags:- - Feedback- parameters: []+ - EvaluationRules+ parameters:+ - name: evaluationRuleId+ in: path+ description: >-+ Stable evaluation-rule identifier returned by the evaluation-rule+ endpoints.+ required: true+ schema:+ type: stringresponses:- '201':+ '200':description: ''content:application/json:schema:- $ref: '#/components/schemas/SubmitFeedbackResponse'+ $ref: '#/components/schemas/EvaluationRule''400':description: ''content:@@ -1894,29 +1803,53 @@ paths:content:application/json:schema: {}- '409':+ '429':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/PublicApiError'+ '500':description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/PublicApiError'security: *ref_0- requestBody:- required: true- content:- application/json:- schema:- $ref: '#/components/schemas/SubmitFeedbackRequest'- /api/public/health:- get:- description: Check health of API and database- operationId: health_health+ patch:+ description: >-+ Update an evaluation rule by its stable identifier.+++ Provide only the fields to change. Providing `evaluatorAssignments`+ replaces the complete assignment list. Replacing the list with an empty+ array disables the rule. Setting `enabled=true` is rejected when the+ resulting assignment list is empty, including when both fields are sent+ in the same request.+++ Legacy trace and dataset rules follow the existing lifecycle+ restrictions: they can be deactivated with `enabled=false`, but their+ name, filters, sampling, and evaluator assignments cannot be changed.+ operationId: evaluationRules_updatetags:- - Health- parameters: []+ - EvaluationRules+ parameters:+ - name: evaluationRuleId+ in: path+ description: >-+ Stable evaluation-rule identifier returned by the evaluation-rule+ endpoints.+ required: true+ schema:+ type: stringresponses:'200':description: ''content:application/json:schema:- $ref: '#/components/schemas/HealthResponse'+ $ref: '#/components/schemas/EvaluationRule''400':description: ''content:@@ -1942,191 +1875,51 @@ paths:content:application/json:schema: {}- '503':- description: ''- /api/public/ingestion:- post:- description: >-- **Deprecated:** On Langfuse Cloud, Langfuse v3 is deprecated and this- endpoint will be removed on November 16, 2026. Send data via the- OpenTelemetry endpoint at `POST /api/public/otel/v1/traces` instead, see- the [OpenTelemetry integration- docs](https://langfuse.com/integrations/native/opentelemetry).- Self-hosted deployments are unaffected by this date; the endpoint- becomes unavailable when they upgrade to Langfuse v4. See the [Langfuse- v3 to v4 upgrade- guide](https://langfuse.com/self-hosting/upgrade/upgrade-guides/upgrade-v3-to-v4).--- **Legacy endpoint for batch ingestion for Langfuse Observability.**--- -> Please use the OpenTelemetry endpoint (`/api/public/otel/v1/traces`).- Learn more: https://langfuse.com/integrations/native/opentelemetry--- Within each batch, there can be multiple events.-- Each event has a type, an id, a timestamp, metadata and a body.-- Internally, we refer to this as the "event envelope" as it tells us- something about the event but not the trace.-- We use the event id within this envelope to deduplicate messages to- avoid processing the same event twice, i.e. the event id should be- unique per request.-- The event.body.id is the ID of the actual trace and will be used for- updates and will be visible within the Langfuse App.-- I.e. if you want to update a trace, you'd use the same body id, but- separate event IDs.--- Notes:-- - Introduction to data model:- https://langfuse.com/docs/observability/data-model-- - Batch sizes are limited to 3.5 MB in total. You need to adjust the- number of events per batch accordingly.-- - The API does not return a 4xx status code for input errors. Instead,- it responds with a 207 status code, which includes a list of the- encountered errors.- operationId: ingestion_batch- tags:- - Ingestion- parameters: []- responses:- '207':+ '409':description: ''content:application/json:schema:- $ref: '#/components/schemas/IngestionResponse'- '400':- description: ''- content:- application/json:- schema: {}- '401':- description: ''- content:- application/json:- schema: {}- '403':- description: ''- content:- application/json:- schema: {}- '404':+ $ref: '#/components/schemas/PublicApiError'+ '429':description: ''content:application/json:- schema: {}- '405':+ schema:+ $ref: '#/components/schemas/PublicApiError'+ '500':description: ''content:application/json:- schema: {}+ schema:+ $ref: '#/components/schemas/PublicApiError'security: *ref_0requestBody:required: truecontent:application/json:schema:- type: object- properties:- batch:- type: array- items:- $ref: '#/components/schemas/IngestionEvent'- description: >-- Batch of tracing events to be ingested. Discriminated by- attribute `type`.- metadata:- nullable: true- description: >-- Optional. Metadata field used by the Langfuse SDKs for- debugging.- required:- - batch- deprecated: true- /api/public/metrics:- get:+ $ref: '#/components/schemas/UpdateEvaluationRuleRequest'+ delete:description: >-- **Deprecated:** On Langfuse Cloud, Langfuse v3 is deprecated and this- endpoint will be removed on November 16, 2026. Use `GET- /api/public/v2/metrics?query=<urlencoded json query>` instead.- Self-hosted deployments are unaffected by this date; the endpoint- becomes unavailable when they upgrade to Langfuse v4. See the [Langfuse- v3 to v4 upgrade- guide](https://langfuse.com/self-hosting/upgrade/upgrade-guides/upgrade-v3-to-v4).--- Get metrics from the Langfuse project using a query object.+ Delete an evaluation rule.- Consider using the [v2 metrics- endpoint](/api-reference#tag/metricsv2/GET/api/public/v2/metrics) for- better performance.+ This removes the live-ingestion rule only. It does not delete associated+ evaluators or scores already produced by them.- For more details, see the [Metrics API- documentation](https://langfuse.com/docs/metrics/features/metrics-api).- operationId: legacy_metricsV1_metrics+ Legacy trace and dataset rules can also be deleted. Their evaluators and+ previously produced scores are preserved.+ operationId: evaluationRules_deletetags:- - LegacyMetricsV1+ - EvaluationRulesparameters:- - name: query- in: query+ - name: evaluationRuleId+ in: pathdescription: >-- JSON string containing the query parameters with the following- structure:-- ```json-- {- "view": string, // Required. One of "traces", "observations", "scores-numeric", "scores-boolean", "scores-categorical"- "dimensions": [ // Optional. Default: []- {- "field": string // Field to group by, e.g. "name", "userId", "sessionId"- }- ],- "metrics": [ // Required. At least one metric must be provided- {- "measure": string, // What to measure, e.g. "count", "latency", "value"- "aggregation": string // How to aggregate, e.g. "count", "sum", "avg", "p95", "histogram"- }- ],- "filters": [ // Optional. Default: []- {- "column": string, // Column to filter on- "operator": string, // Operator, e.g. "=", ">", "<", "contains"- "value": any, // Value to compare against- "type": string, // Data type, e.g. "string", "number", "stringObject"- "key": string // Required only when filtering on metadata- }- ],- "timeDimension": { // Optional. Default: null. If provided, results will be grouped by time- "granularity": string // One of "minute", "hour", "day", "week", "month", "auto"- },- "fromTimestamp": string, // Required. ISO datetime string for start of time range- "toTimestamp": string, // Required. ISO datetime string for end of time range- "orderBy": [ // Optional. Default: null- {- "field": string, // Field to order by- "direction": string // "asc" or "desc"- }- ],- "config": { // Optional. Query-specific configuration- "bins": number, // Optional. Number of bins for histogram (1-100), default: 10- "row_limit": number // Optional. Row limit for results (1-1000)- }- }-- ```+ Stable evaluation-rule identifier returned by the evaluation-rule+ endpoints.required: trueschema:type: string@@ -2136,7 +1929,7 @@ paths:content:application/json:schema:- $ref: '#/components/schemas/legacyMetricsResponse'+ $ref: '#/components/schemas/DeletedEvaluationRule''400':description: ''content:@@ -2162,40 +1955,44 @@ paths:content:application/json:schema: {}+ '429':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/PublicApiError'+ '500':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/PublicApiError'security: *ref_0- deprecated: true- /api/public/observations/{observationId}:- get:+ /api/public/v2/evaluators:+ post:description: >-- **Deprecated:** On Langfuse Cloud, Langfuse v3 is deprecated and this- endpoint will be removed on November 16, 2026. Use `GET- /api/public/v2/observations?fromStartTime=<from>&toStartTime=<to>`- instead. Self-hosted deployments are unaffected by this date; the- endpoint becomes unavailable when they upgrade to Langfuse v4. See the- [Langfuse v3 to v4 upgrade- guide](https://langfuse.com/self-hosting/upgrade/upgrade-guides/upgrade-v3-to-v4).+ Create an evaluator in the authenticated project.- Get a observation- operationId: legacy_observationsV1_get+ An evaluator defines **how** Langfuse should score data. LLM-as-a-judge+ evaluators define a prompt, expected structured output, and optional+ model configuration. Code evaluators define source code and a runtime+ language.+++ This always creates a new evaluator with version `1`. Names are not+ identifiers and do not need to be unique.+ operationId: evaluators_createtags:- - LegacyObservationsV1- parameters:- - name: observationId- in: path- description: >-- The unique langfuse identifier of an observation, can be an event,- span or generation- required: true- schema:- type: string+ - Evaluators+ parameters: []responses:- '200':+ '201':description: ''content:application/json:schema:- $ref: '#/components/schemas/ObservationsViewSingle'+ $ref: '#/components/schemas/Evaluator''400':description: ''content:@@ -2221,268 +2018,55 @@ paths:content:application/json:schema: {}+ '412':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/PublicApiError'+ '429':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/PublicApiError'+ '500':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/PublicApiError'security: *ref_0- deprecated: true- /api/public/observations:+ requestBody:+ required: true+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/CreateEvaluatorRequest'get:description: >-- **Deprecated:** On Langfuse Cloud, Langfuse v3 is deprecated and this- endpoint will be removed on November 16, 2026. Use `GET- /api/public/v2/observations?fromStartTime=<from>&toStartTime=<to>`- instead. Self-hosted deployments are unaffected by this date; the- endpoint becomes unavailable when they upgrade to Langfuse v4. See the- [Langfuse v3 to v4 upgrade- guide](https://langfuse.com/self-hosting/upgrade/upgrade-guides/upgrade-v3-to-v4).+ List evaluators in newest-first creation order.- Get a list of observations.--- Consider using the [v2 observations- endpoint](/api-reference#tag/observationsv2/GET/api/public/v2/observations)- for cursor-based pagination and field selection.- operationId: legacy_observationsV1_getMany+ Every evaluator includes its latest definition and version metadata+ flattened into the evaluator object, plus associated evaluation rules.+ Treat the cursor as opaque and return it unchanged.+ operationId: evaluators_listtags:- - LegacyObservationsV1+ - Evaluatorsparameters:- - name: page- in: query- description: Page number, starts at 1.- required: false- schema:- type: integer- nullable: true- name: limitin: querydescription: >-- Limit of items per page. If you encounter api issues due to too- large page sizes, try to reduce the limit.+ Maximum number of items to return. Defaults to `50` and cannot+ exceed `100`.required: falseschema:type: integernullable: true- - name: name- in: query- required: false- schema:- type: string- nullable: true- - name: userId- in: query- required: false- schema:- type: string- nullable: true- - name: type- in: query- required: false- schema:- type: string- nullable: true- - name: traceId- in: query- required: false- schema:- type: string- nullable: true- - name: level- in: query- description: >-- Optional filter for observations with a specific level (e.g.- "DEBUG", "DEFAULT", "WARNING", "ERROR").- required: false- schema:- $ref: '#/components/schemas/ObservationLevel'- nullable: true- - name: parentObservationId- in: query- required: false- schema:- type: string- nullable: true- - name: environment- in: query- description: >-- Optional filter for observations where the environment is one of the- provided values.- required: false- schema:- type: array- items:- type: string- nullable: true- - name: fromStartTime- in: query- description: >-- Retrieve only observations with a start_time on or after this- datetime (ISO 8601).- required: false- schema:- type: string- format: date-time- nullable: true- - name: toStartTime- in: query- description: >-- Retrieve only observations with a start_time before this datetime- (ISO 8601).- required: false- schema:- type: string- format: date-time- nullable: true- - name: version- in: query- description: Optional filter to only include observations with a certain version.- required: false- schema:- type: string- nullable: true- - name: filter+ - name: cursorin: query- description: >-- JSON string containing an array of filter conditions. When provided,- this takes precedence over query parameter filters (userId, name,- type, level, environment, fromStartTime, ...).--- ## Filter Structure-- Each filter condition has the following structure:-- ```json-- [- {- "type": string, // Required. One of: "datetime", "string", "number", "stringOptions", "categoryOptions", "arrayOptions", "stringObject", "numberObject", "boolean", "null"- "column": string, // Required. Column to filter on (see available columns below)- "operator": string, // Required. Operator based on type:- // - datetime: ">", "<", ">=", "<="- // - string: "=", "contains", "does not contain", "starts with", "ends with"- // - stringOptions: "any of", "none of"- // - categoryOptions: "any of", "none of"- // - arrayOptions: "any of", "none of", "all of"- // - number: "=", ">", "<", ">=", "<="- // - stringObject: "=", "contains", "does not contain", "starts with", "ends with"- // - numberObject: "=", ">", "<", ">=", "<="- // - boolean: "=", "<>"- // - null: "is null", "is not null"- "value": any, // Required (except for null type). Value to compare against. Type depends on filter type- "key": string // Required only for stringObject, numberObject, and categoryOptions types when filtering on nested fields like metadata- }- ]-- ```--- ## Available Columns--- ### Core Observation Fields-- - `id` (string) - Observation ID-- - `type` (string) - Observation type (SPAN, GENERATION, EVENT)-- - `name` (string) - Observation name-- - `traceId` (string) - Associated trace ID-- - `startTime` (datetime) - Observation start time-- - `endTime` (datetime) - Observation end time-- - `environment` (string) - Environment tag-- - `level` (string) - Log level (DEBUG, DEFAULT, WARNING, ERROR)-- - `statusMessage` (string) - Status message-- - `version` (string) - Version tag--- ### Performance Metrics-- - `latency` (number) - Latency in seconds (calculated: end_time -- start_time)-- - `timeToFirstToken` (number) - Time to first token in seconds-- - `tokensPerSecond` (number) - Output tokens per second--- ### Token Usage-- - `inputTokens` (number) - Number of input tokens-- - `outputTokens` (number) - Number of output tokens-- - `totalTokens` (number) - Total tokens (alias: `tokens`)--- ### Cost Metrics-- - `inputCost` (number) - Input cost in USD-- - `outputCost` (number) - Output cost in USD-- - `totalCost` (number) - Total cost in USD--- ### Model Information-- - `model` (string) - Provided model name-- - `promptName` (string) - Associated prompt name-- - `promptVersion` (number) - Associated prompt version--- ### Structured Data-- - `metadata` (stringObject/numberObject/categoryOptions) - Metadata- key-value pairs. Use `key` parameter to filter on specific metadata- keys.--- ### Associated Trace Fields (requires join with traces table)-- - `userId` (string) - User ID from associated trace-- - `traceName` (string) - Name from associated trace-- - `traceEnvironment` (string) - Environment from associated trace-- - `traceTags` (arrayOptions) - Tags from associated trace--- ## Filter Examples-- ```json-- [- {- "type": "string",- "column": "type",- "operator": "=",- "value": "GENERATION"- },- {- "type": "number",- "column": "latency",- "operator": ">=",- "value": 2.5- },- {- "type": "stringObject",- "column": "metadata",- "key": "environment",- "operator": "=",- "value": "production"- }- ]-- ```+ description: Opaque cursor returned by the previous page.required: falseschema:type: string@@ -2493,7 +2077,7 @@ paths:content:application/json:schema:- $ref: '#/components/schemas/legacyObservationsViews'+ $ref: '#/components/schemas/EvaluatorsPage''400':description: ''content:@@ -2519,24 +2103,46 @@ paths:content:application/json:schema: {}+ '429':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/PublicApiError'+ '500':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/PublicApiError'security: *ref_0- deprecated: true- /api/public/scores/{scoreId}:- delete:- description: Delete a score (supports both trace and session scores)- operationId: legacy_scoreV1_delete+ /api/public/v2/evaluators/{evaluatorId}:+ get:+ description: >-+ Get one evaluator by its stable identifier.+++ The response includes the evaluator's latest definition and version+ metadata flattened into the evaluator object, plus associated evaluation+ rules. Use the version-history endpoint when older definitions are+ needed.+ operationId: evaluators_gettags:- - LegacyScoreV1+ - Evaluatorsparameters:- - name: scoreId+ - name: evaluatorIdin: path- description: The unique langfuse identifier of a score+ description: Stable evaluator identifier returned by the evaluator endpoints.required: trueschema:type: stringresponses:- '204':+ '200':description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/Evaluator''400':description: ''content:@@ -2562,35 +2168,58 @@ paths:content:application/json:schema: {}+ '429':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/PublicApiError'+ '500':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/PublicApiError'security: *ref_0- /api/public/llm-connections:- get:- description: Get all LLM connections in a project- operationId: llmConnections_list+ patch:+ description: >-+ Update an evaluator by its stable identifier.+++ Provide only the top-level fields to change. Metadata-only changes do+ not create a version. Evaluator type cannot change.+++ Definition fields are flattened into the request. To replace a+ definition, include `type` and every definition field for that type.+ Definition fields are replaced as a complete unit rather than merged.+ For LLM-as-a-judge evaluators, omitting or setting `modelConfig` to+ `null` selects the project's default evaluation model.+++ Replacing a definition automatically returns an evaluator paused by an+ invalid or missing model configuration to `active`, while preserving its+ id and evaluation-rule assignments. Pauses caused by provider+ authentication, billing, connectivity, account state, or an unknown+ legacy reason remain paused until the explicit reactivation check+ succeeds.+ operationId: evaluators_updatetags:- - LlmConnections+ - Evaluatorsparameters:- - name: page- in: query- description: page number, starts at 1- required: false- schema:- type: integer- nullable: true- - name: limit- in: query- description: limit of items per page- required: false+ - name: evaluatorId+ in: path+ description: Stable evaluator identifier returned by the evaluator endpoints.+ required: trueschema:- type: integer- nullable: true+ type: stringresponses:'200':description: ''content:application/json:schema:- $ref: '#/components/schemas/PaginatedLlmConnections'+ $ref: '#/components/schemas/Evaluator''400':description: ''content:@@ -2616,65 +2245,51 @@ paths:content:application/json:schema: {}- security: *ref_0- put:- description: >-- Create or update an LLM connection. The connection is upserted on- provider.- operationId: llmConnections_upsert- tags:- - LlmConnections- parameters: []- responses:- '200':+ '409':description: ''content:application/json:schema:- $ref: '#/components/schemas/LlmConnection'- '400':- description: ''- content:- application/json:- schema: {}- '401':- description: ''- content:- application/json:- schema: {}- '403':+ $ref: '#/components/schemas/PublicApiError'+ '412':description: ''content:application/json:- schema: {}- '404':+ schema:+ $ref: '#/components/schemas/PublicApiError'+ '429':description: ''content:application/json:- schema: {}- '405':+ schema:+ $ref: '#/components/schemas/PublicApiError'+ '500':description: ''content:application/json:- schema: {}+ schema:+ $ref: '#/components/schemas/PublicApiError'security: *ref_0requestBody:required: truecontent:application/json:schema:- $ref: '#/components/schemas/UpsertLlmConnectionRequest'- /api/public/llm-connections/{id}:+ $ref: '#/components/schemas/UpdateEvaluatorRequest'delete:description: >-- Delete an LLM connection by id. Evaluators that depend on the deleted- connection are automatically paused.- operationId: llmConnections_delete+ Delete an evaluator and all of its stored versions.+++ Associated evaluation-rule assignments are also removed. Scores already+ produced by the evaluator are preserved.+ operationId: evaluators_deletetags:- - LlmConnections+ - Evaluatorsparameters:- - name: id+ - name: evaluatorIdin: path+ description: Stable evaluator identifier returned by the evaluator endpoints.required: trueschema:type: string@@ -2684,7 +2299,7 @@ paths:content:application/json:schema:- $ref: '#/components/schemas/DeleteLlmConnectionResponse'+ $ref: '#/components/schemas/DeletedEvaluator''400':description: ''content:@@ -2710,27 +2325,61 @@ paths:content:application/json:schema: {}+ '429':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/PublicApiError'+ '500':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/PublicApiError'security: *ref_0- /api/public/media/{mediaId}:+ /api/public/v2/evaluators/{evaluatorId}/versions:get:- description: Get a media record- operationId: media_get+ description: >-+ List an evaluator's version history in newest-first order.+++ This endpoint is intended for history and audit use cases. Ordinary+ clients can use the flattened `version` and definition fields on the+ evaluator response.+ operationId: evaluators_listVersionstags:- - Media+ - Evaluatorsparameters:- - name: mediaId+ - name: evaluatorIdin: path- description: The unique langfuse identifier of a media record+ description: Stable evaluator identifier returned by the evaluator endpoints.required: trueschema:type: string+ - name: limit+ in: query+ description: >-+ Maximum number of versions to return. Defaults to `50` and cannot+ exceed `100`.+ required: false+ schema:+ type: integer+ nullable: true+ - name: cursor+ in: query+ description: Opaque cursor returned by the previous page.+ required: false+ schema:+ type: string+ nullable: trueresponses:'200':description: ''content:application/json:schema:- $ref: '#/components/schemas/GetMediaResponse'+ $ref: '#/components/schemas/EvaluatorVersionsPage''400':description: ''content:@@ -2756,22 +2405,112 @@ paths:content:application/json:schema: {}+ '429':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/PublicApiError'+ '500':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/PublicApiError'security: *ref_0- patch:- description: Patch a media record- operationId: media_patch+ /api/public/experiments:+ get:+ description: |-+ List experiments with cursor-based pagination. Results are ordered by+ latest experiment activity descending.+ operationId: experiments_listtags:- - Media+ - Experimentsparameters:- - name: mediaId- in: path- description: The unique langfuse identifier of a media record+ - name: fields+ in: query+ description: |-+ Comma-separated list of field groups to include. Available groups:+ `core`, `metadata`, `scores`. If omitted, `core` is returned.+ required: false+ schema:+ type: string+ nullable: true+ - name: limit+ in: query+ description: Number of experiments to return per page. Maximum 100, default 50.+ required: false+ schema:+ type: integer+ nullable: true+ - name: scoreLimit+ in: query+ description: >-+ Number of scores to return per experiment when `fields=scores` is+ requested. Maximum 50, default 50.+ required: false+ schema:+ type: integer+ nullable: true+ - name: cursor+ in: query+ description: Versioned base64url cursor from the previous response page.+ required: false+ schema:+ type: string+ nullable: true+ - name: fromStartTime+ in: query+ description: Retrieve only experiments on or after this datetime.required: trueschema:type: string+ format: date-time+ - name: toStartTime+ in: query+ description: Retrieve only experiments before this datetime.+ required: false+ schema:+ type: string+ format: date-time+ nullable: true+ - name: id+ in: query+ description: Comma-separated list of experiment IDs.+ required: false+ schema:+ type: string+ nullable: true+ - name: name+ in: query+ description: Comma-separated list of experiment names.+ required: false+ schema:+ type: string+ nullable: true+ - name: datasetId+ in: query+ description: Comma-separated list of dataset IDs.+ required: false+ schema:+ type: string+ nullable: true+ - name: filter+ in: query+ description: |-+ JSON string containing an array of structured filter conditions.+ Supported columns are `id`, `name`, and `datasetId`.+ required: false+ schema:+ type: string+ nullable: trueresponses:- '204':+ '200':description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/ExperimentsResponse''400':description: ''content:@@ -2798,26 +2537,159 @@ paths:application/json:schema: {}security: *ref_0- requestBody:- required: true- content:- application/json:- schema:- $ref: '#/components/schemas/PatchMediaBody'- /api/public/media:- post:- description: Get a presigned upload URL for a media record- operationId: media_getUploadUrl- tags:- - Media- parameters: []- responses:- '200':- description: ''+ /api/public/experiment-items:+ get:+ description: |-+ List experiment items with cursor-based pagination. Use this endpoint+ to export experiment item inputs, outputs, expected outputs, metadata,+ and optionally item/trace scores. Results are ordered by time+ descending.+ operationId: experiments_listItems+ tags:+ - Experiments+ parameters:+ - name: fields+ in: query+ description: |-+ Comma-separated list of field groups to include. Available groups:+ `core`, `dataset`, `io`, `metadata`, `itemMetadata`,+ `experimentMetadata`, `scores`. If omitted, `core,dataset` is+ returned.+ required: false+ schema:+ type: string+ nullable: true+ - name: limit+ in: query+ description: >-+ Number of experiment items to return per page. Maximum 100, default+ 50.+ required: false+ schema:+ type: integer+ nullable: true+ - name: scoreLimit+ in: query+ description: >-+ Number of scores to return per experiment item when `fields=scores`+ is requested. Maximum 50, default 50.+ required: false+ schema:+ type: integer+ nullable: true+ - name: cursor+ in: query+ description: Versioned base64url cursor from the previous response page.+ required: false+ schema:+ type: string+ nullable: true+ - name: fromStartTime+ in: query+ description: Retrieve only experiment items started on or after this datetime.+ required: true+ schema:+ type: string+ format: date-time+ - name: toStartTime+ in: query+ description: Retrieve only experiment items started before this datetime.+ required: false+ schema:+ type: string+ format: date-time+ nullable: true+ - name: experimentId+ in: query+ description: Comma-separated list of experiment IDs.+ required: false+ schema:+ type: string+ nullable: true+ - name: experimentName+ in: query+ description: Comma-separated list of experiment names.+ required: false+ schema:+ type: string+ nullable: true+ - name: experimentItemId+ in: query+ description: Comma-separated list of experiment item IDs.+ required: false+ schema:+ type: string+ nullable: true+ - name: datasetId+ in: query+ description: Comma-separated list of dataset IDs.+ required: false+ schema:+ type: string+ nullable: true+ - name: filter+ in: query+ description: |-+ JSON string containing an array of structured filter conditions.+ Supported columns are `experimentId`, `experimentName`,+ `experimentItemId`, and `datasetId`.+ required: false+ schema:+ type: string+ nullable: true+ responses:+ '200':+ description: ''content:application/json:schema:- $ref: '#/components/schemas/GetMediaUploadUrlResponse'+ $ref: '#/components/schemas/ExperimentItemsResponse'+ '400':+ description: ''+ content:+ application/json:+ schema: {}+ '401':+ description: ''+ content:+ application/json:+ schema: {}+ '403':+ description: ''+ content:+ application/json:+ schema: {}+ '404':+ description: ''+ content:+ application/json:+ schema: {}+ '405':+ description: ''+ content:+ application/json:+ schema: {}+ security: *ref_0+ /api/public/feedback:+ post:+ description: >-+ Submit explicit user-approved feedback about Langfuse skills, MCP tools,+ CLI, docs, or public API. Do not include secrets, credentials, customer+ data, trace payloads, or unrelated use-case details.+++ Not available in the HIPAA cloud region.+ operationId: feedback_submit+ tags:+ - Feedback+ parameters: []+ responses:+ '201':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/SubmitFeedbackResponse''400':description: ''content:@@ -2843,243 +2715,190 @@ paths:content:application/json:schema: {}+ '409':+ description: ''security: *ref_0requestBody:required: truecontent:application/json:schema:- $ref: '#/components/schemas/GetMediaUploadUrlRequest'- /api/public/v2/metrics:+ $ref: '#/components/schemas/SubmitFeedbackRequest'+ /api/public/health:get:+ description: Check health of API and database+ operationId: health_health+ tags:+ - Health+ parameters: []+ responses:+ '200':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/HealthResponse'+ '400':+ description: ''+ content:+ application/json:+ schema: {}+ '401':+ description: ''+ content:+ application/json:+ schema: {}+ '403':+ description: ''+ content:+ application/json:+ schema: {}+ '404':+ description: ''+ content:+ application/json:+ schema: {}+ '405':+ description: ''+ content:+ application/json:+ schema: {}+ '503':+ description: ''+ /api/public/ingestion:+ post:description: >-- Get metrics from the Langfuse project using a query object. V2 endpoint- with optimized performance.--- ## V2 Differences-- - Supports `observations`, `scores-numeric`, `scores-boolean`, and- `scores-categorical` views only (traces view not supported)-- - Direct access to tags and release fields on observations-- - Semantic-root filtering and grouping through the v2-only- `isRootObservation` dimension-- - Backwards-compatible: traceName, traceRelease, traceVersion dimensions- are still available on observations view+ **Deprecated:** On Langfuse Cloud, Langfuse v3 is deprecated and this+ endpoint will be removed on November 16, 2026. Send data via the+ OpenTelemetry endpoint at `POST /api/public/otel/v1/traces` instead, see+ the [OpenTelemetry integration+ docs](https://langfuse.com/integrations/native/opentelemetry).+ Self-hosted deployments are unaffected by this date; the endpoint+ becomes unavailable when they upgrade to Langfuse v4. See the [Langfuse+ v3 to v4 upgrade+ guide](https://langfuse.com/self-hosting/upgrade/upgrade-guides/upgrade-v3-to-v4).- - High cardinality dimensions are not supported and will return a 400- error (see below)+ **Legacy endpoint for batch ingestion for Langfuse Observability.**- For more details, see the [Metrics API- documentation](https://langfuse.com/docs/metrics/features/metrics-api).+ -> Please use the OpenTelemetry endpoint (`/api/public/otel/v1/traces`).+ Learn more: https://langfuse.com/integrations/native/opentelemetry- ## Available Views+ Within each batch, there can be multiple events.- ### observations+ Each event has a type, an id, a timestamp, metadata and a body.- Query observation-level data (spans, generations, events).+ Internally, we refer to this as the "event envelope" as it tells us+ something about the event but not the trace.+ We use the event id within this envelope to deduplicate messages to+ avoid processing the same event twice, i.e. the event id should be+ unique per request.- **Dimensions:**+ The event.body.id is the ID of the actual trace and will be used for+ updates and will be visible within the Langfuse App.- - `environment` - Deployment environment (e.g., production, staging)+ I.e. if you want to update a trace, you'd use the same body id, but+ separate event IDs.- - `type` - Type of observation (SPAN, GENERATION, EVENT)- - `name` - Name of the observation+ Notes:- - `level` - Logging level of the observation+ - Introduction to data model:+ https://langfuse.com/docs/observability/data-model- - `version` - Version of the observation-- - `tags` - User-defined tags-- - `release` - Release version-- - `traceName` - Name of the parent trace (backwards-compatible)-- - `traceRelease` - Release version of the parent trace- (backwards-compatible, maps to release)-- - `traceVersion` - Version of the parent trace (backwards-compatible,- maps to version)-- - `providedModelName` - Name of the model used-- - `promptName` - Name of the prompt used-- - `promptVersion` - Version of the prompt used-- - `isRootObservation` - Boolean semantic-root status. `true` includes- physical roots and app roots whose SDK parent is external (so- `parentObservationId` may be non-null).-- - `startTimeMonth` - Month of start_time in YYYY-MM format--- **Measures:**-- - `count` - Total number of observations-- - `latency` - Observation latency (milliseconds)-- - `streamingLatency` - Generation latency from completion start to end- (milliseconds)-- - `inputTokens` - Sum of input tokens consumed-- - `outputTokens` - Sum of output tokens produced-- - `totalTokens` - Sum of all tokens consumed-- - `outputTokensPerSecond` - Output tokens per second-- - `tokensPerSecond` - Total tokens per second-- - `inputCost` - Input cost (USD)-- - `outputCost` - Output cost (USD)-- - `totalCost` - Total cost (USD)-- - `timeToFirstToken` - Time to first token (milliseconds)-- - `countScores` - Number of scores attached to the observation--- ### scores-numeric-- Query numeric and boolean score data.--- **Dimensions:**-- - `environment` - Deployment environment-- - `name` - Name of the score (e.g., accuracy, toxicity)-- - `source` - Origin of the score (API, ANNOTATION, EVAL)-- - `dataType` - Data type (NUMERIC, BOOLEAN)-- - `configId` - Identifier of the score config-- - `timestampMonth` - Month in YYYY-MM format-- - `timestampDay` - Day in YYYY-MM-DD format-- - `value` - Numeric value of the score-- - `traceName` - Name of the parent trace-- - `tags` - Tags-- - `traceRelease` - Release version-- - `traceVersion` - Version-- - `observationName` - Name of the associated observation-- - `observationModelName` - Model name of the associated observation-- - `observationPromptName` - Prompt name of the associated observation-- - `observationPromptVersion` - Prompt version of the associated- observation--- **Measures:**-- - `count` - Total number of scores-- - `value` - Score value (for aggregations)--- ### scores-boolean-- Query boolean score data. It has the same score and parent- trace/observation dimensions as scores-numeric, plus:--- **Dimensions:**-- - `booleanValue` - Boolean value for true/false grouping and filtering--- **Measures:**-- - `count` - Total number of boolean scores-- - `value` - Numeric 0/1 score value; `avg` returns the true-rate--- ### scores-categorical-- Query categorical score data. Same dimensions as scores-numeric except- uses `stringValue` instead of `value`.--- **Measures:**-- - `count` - Total number of scores--- ## High Cardinality Dimensions-- The following dimensions cannot be used as grouping dimensions in v2- metrics API as they can cause performance issues.-- Use them in filters instead.--- **observations view:**-- - `id` - Use traceId filter to narrow down results-- - `traceId` - Use traceId filter instead-- - `userId` - Use userId filter instead-- - `sessionId` - Use sessionId filter instead-- - `parentObservationId` - Use parentObservationId filter instead--- **scores-numeric / scores-boolean / scores-categorical views:**-- - `id` - Use specific filters to narrow down results-- - `traceId` - Use traceId filter instead-- - `userId` - Use userId filter instead-- - `sessionId` - Use sessionId filter instead-- - `observationId` - Use observationId filter instead+ - Batch sizes are limited to 3.5 MB in total. You need to adjust the+ number of events per batch accordingly.+ - The API does not return a 4xx status code for input errors. Instead,+ it responds with a 207 status code, which includes a list of the+ encountered errors.+ operationId: ingestion_batch+ tags:+ - Ingestion+ parameters: []+ responses:+ '207':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/IngestionResponse'+ '400':+ description: ''+ content:+ application/json:+ schema: {}+ '401':+ description: ''+ content:+ application/json:+ schema: {}+ '403':+ description: ''+ content:+ application/json:+ schema: {}+ '404':+ description: ''+ content:+ application/json:+ schema: {}+ '405':+ description: ''+ content:+ application/json:+ schema: {}+ security: *ref_0+ requestBody:+ required: true+ content:+ application/json:+ schema:+ type: object+ properties:+ batch:+ type: array+ items:+ $ref: '#/components/schemas/IngestionEvent'+ description: >-+ Batch of tracing events to be ingested. Discriminated by+ attribute `type`.+ metadata:+ nullable: true+ description: >-+ Optional. Metadata field used by the Langfuse SDKs for+ debugging.+ required:+ - batch+ deprecated: true+ /api/public/metrics:+ get:+ description: >-+ **Deprecated:** On Langfuse Cloud, Langfuse v3 is deprecated and this+ endpoint will be removed on November 16, 2026. Use `GET+ /api/public/v2/metrics?query=<urlencoded json query>` instead.+ Self-hosted deployments are unaffected by this date; the endpoint+ becomes unavailable when they upgrade to Langfuse v4. See the [Langfuse+ v3 to v4 upgrade+ guide](https://langfuse.com/self-hosting/upgrade/upgrade-guides/upgrade-v3-to-v4).- ## Aggregations- Available aggregation functions: `sum`, `avg`, `count`, `max`, `min`,- `p50`, `p75`, `p90`, `p95`, `p99`, `histogram`+ Get metrics from the Langfuse project using a query object.- ## Time Granularities+ Consider using the [v2 metrics+ endpoint](/api-reference#tag/metricsv2/GET/api/public/v2/metrics) for+ better performance.- Available granularities for timeDimension: `auto`, `minute`, `hour`,- `day`, `week`, `month`- - `auto` bins the data into approximately 50 buckets based on the time- range- operationId: metrics_metrics+ For more details, see the [Metrics API+ documentation](https://langfuse.com/docs/metrics/features/metrics-api).+ operationId: legacy_metricsV1_metricstags:- - Metrics+ - LegacyMetricsV1parameters:- name: queryin: query@@ -3090,69 +2909,45 @@ paths:```json{- "view": string, // Required. One of "observations", "scores-numeric", "scores-boolean", "scores-categorical"+ "view": string, // Required. One of "traces", "observations", "scores-numeric", "scores-boolean", "scores-categorical""dimensions": [ // Optional. Default: []{- "field": string // Field to group by (see available dimensions above)+ "field": string // Field to group by, e.g. "name", "userId", "sessionId"}],"metrics": [ // Required. At least one metric must be provided{- "measure": string, // What to measure (see available measures above)- "aggregation": string // How to aggregate: "sum", "avg", "count", "max", "min", "p50", "p75", "p90", "p95", "p99", "histogram"+ "measure": string, // What to measure, e.g. "count", "latency", "value"+ "aggregation": string // How to aggregate, e.g. "count", "sum", "avg", "p95", "histogram"}],"filters": [ // Optional. Default: []{- "column": string, // Column to filter on (any dimension field)- "operator": string, // Operator based on type:- // - datetime: ">", "<", ">=", "<="- // - string: "=", "contains", "does not contain", "starts with", "ends with"- // - stringOptions: "any of", "none of"- // - arrayOptions: "any of", "none of", "all of"- // - number: "=", ">", "<", ">=", "<="- // - stringObject/numberObject: same as string/number with required "key"- // - boolean: "=", "<>"- // - null: "is null", "is not null"+ "column": string, // Column to filter on+ "operator": string, // Operator, e.g. "=", ">", "<", "contains""value": any, // Value to compare against- "type": string, // Data type: "datetime", "string", "number", "stringOptions", "categoryOptions", "arrayOptions", "stringObject", "numberObject", "boolean", "null"- "key": string // Required only for stringObject/numberObject types (e.g., metadata filtering)+ "type": string, // Data type, e.g. "string", "number", "stringObject"+ "key": string // Required only when filtering on metadata}],"timeDimension": { // Optional. Default: null. If provided, results will be grouped by time- "granularity": string // One of "auto", "minute", "hour", "day", "week", "month"+ "granularity": string // One of "minute", "hour", "day", "week", "month", "auto"},"fromTimestamp": string, // Required. ISO datetime string for start of time range- "toTimestamp": string, // Required. ISO datetime string for end of time range (must be after fromTimestamp)+ "toTimestamp": string, // Required. ISO datetime string for end of time range"orderBy": [ // Optional. Default: null{- "field": string, // Field to order by (dimension or metric alias)+ "field": string, // Field to order by"direction": string // "asc" or "desc"}],"config": { // Optional. Query-specific configuration- "bins": number, // Optional. Number of bins for histogram aggregation (1-100), default: 10- "row_limit": number // Optional. Maximum number of rows to return (1-1000), default: 100+ "bins": number, // Optional. Number of bins for histogram (1-100), default: 10+ "row_limit": number // Optional. Row limit for results (1-1000)}}```--- For example, to count semantic roots (including app roots with a- non-null external parent), use a boolean filter:-- ```json-- {- "view": "observations",- "metrics": [{"measure": "count", "aggregation": "count"}],- "filters": [{"column": "isRootObservation", "operator": "=", "value": true, "type": "boolean"}],- "fromTimestamp": "2025-01-01T00:00:00.000Z",- "toTimestamp": "2025-02-01T00:00:00.000Z"- }-- ```required: trueschema:type: string@@ -3162,7 +2957,7 @@ paths:content:application/json:schema:- $ref: '#/components/schemas/MetricsV2Response'+ $ref: '#/components/schemas/legacyMetricsResponse''400':description: ''content:@@ -3189,20 +2984,39 @@ paths:application/json:schema: {}security: *ref_0- /api/public/models:- post:- description: Create a model- operationId: models_create+ deprecated: true+ /api/public/observations/{observationId}:+ get:+ description: >-+ **Deprecated:** On Langfuse Cloud, Langfuse v3 is deprecated and this+ endpoint will be removed on November 16, 2026. Use `GET+ /api/public/v2/observations?fromStartTime=<from>&toStartTime=<to>`+ instead. Self-hosted deployments are unaffected by this date; the+ endpoint becomes unavailable when they upgrade to Langfuse v4. See the+ [Langfuse v3 to v4 upgrade+ guide](https://langfuse.com/self-hosting/upgrade/upgrade-guides/upgrade-v3-to-v4).+++ Get a observation+ operationId: legacy_observationsV1_gettags:- - Models- parameters: []+ - LegacyObservationsV1+ parameters:+ - name: observationId+ in: path+ description: >-+ The unique langfuse identifier of an observation, can be an event,+ span or generation+ required: true+ schema:+ type: stringresponses:'200':description: ''content:application/json:schema:- $ref: '#/components/schemas/Model'+ $ref: '#/components/schemas/ObservationsViewSingle''400':description: ''content:@@ -3229,343 +3043,59 @@ paths:application/json:schema: {}security: *ref_0- requestBody:- required: true- content:- application/json:- schema:- $ref: '#/components/schemas/CreateModelRequest'+ deprecated: true+ /api/public/observations:get:- description: Get all models- operationId: models_list+ description: >-+ **Deprecated:** On Langfuse Cloud, Langfuse v3 is deprecated and this+ endpoint will be removed on November 16, 2026. Use `GET+ /api/public/v2/observations?fromStartTime=<from>&toStartTime=<to>`+ instead. Self-hosted deployments are unaffected by this date; the+ endpoint becomes unavailable when they upgrade to Langfuse v4. See the+ [Langfuse v3 to v4 upgrade+ guide](https://langfuse.com/self-hosting/upgrade/upgrade-guides/upgrade-v3-to-v4).+++ Get a list of observations.+++ Consider using the [v2 observations+ endpoint](/api-reference#tag/observationsv2/GET/api/public/v2/observations)+ for cursor-based pagination and field selection.+ operationId: legacy_observationsV1_getManytags:- - Models+ - LegacyObservationsV1parameters:- name: pagein: query- description: page number, starts at 1+ description: Page number, starts at 1.required: falseschema:type: integernullable: true- name: limitin: query- description: limit of items per page+ description: >-+ Limit of items per page. If you encounter api issues due to too+ large page sizes, try to reduce the limit.required: falseschema:type: integernullable: true- responses:- '200':- description: ''- content:- application/json:- schema:- $ref: '#/components/schemas/PaginatedModels'- '400':- description: ''- content:- application/json:- schema: {}- '401':- description: ''- content:- application/json:- schema: {}- '403':- description: ''- content:- application/json:- schema: {}- '404':- description: ''- content:- application/json:- schema: {}- '405':- description: ''- content:- application/json:- schema: {}- security: *ref_0- /api/public/models/{id}:- put:- description: >-- Create or replace a project-owned model using its id. Built-in models- cannot be modified.- operationId: models_upsert- tags:- - Models- parameters:- - name: id- in: path- required: true+ - name: name+ in: query+ required: falseschema:type: string- responses:- '200':- description: ''- content:- application/json:- schema:- $ref: '#/components/schemas/Model'- '400':- description: ''- content:- application/json:- schema: {}- '401':- description: ''- content:- application/json:- schema: {}- '403':- description: ''- content:- application/json:- schema: {}- '404':- description: ''- content:- application/json:- schema: {}- '405':- description: ''- content:- application/json:- schema: {}- security: *ref_0- requestBody:- required: true- content:- application/json:- schema:- $ref: '#/components/schemas/CreateModelRequest'- get:- description: Get a model- operationId: models_get- tags:- - Models- parameters:- - name: id- in: path- required: true- schema:- type: string- responses:- '200':- description: ''- content:- application/json:- schema:- $ref: '#/components/schemas/Model'- '400':- description: ''- content:- application/json:- schema: {}- '401':- description: ''- content:- application/json:- schema: {}- '403':- description: ''- content:- application/json:- schema: {}- '404':- description: ''- content:- application/json:- schema: {}- '405':- description: ''- content:- application/json:- schema: {}- security: *ref_0- delete:- description: >-- Delete a model. Cannot delete models managed by Langfuse. You can create- your own definition with the same modelName to override the definition- though.- operationId: models_delete- tags:- - Models- parameters:- - name: id- in: path- required: true- schema:- type: string- responses:- '204':- description: ''- '400':- description: ''- content:- application/json:- schema: {}- '401':- description: ''- content:- application/json:- schema: {}- '403':- description: ''- content:- application/json:- schema: {}- '404':- description: ''- content:- application/json:- schema: {}- '405':- description: ''- content:- application/json:- schema: {}- security: *ref_0- /api/public/v2/observations:- get:- description: >-- Get a list of observations with cursor-based pagination and flexible- field selection.--- ## Cursor-based Pagination-- This endpoint uses cursor-based pagination for efficient traversal of- large datasets.-- The cursor is returned in the response metadata and should be passed in- subsequent requests-- to retrieve the next page of results.--- ## Field Selection-- Use the `fields` parameter to control which observation fields are- returned:-- - `core` - Always included: id, traceId, startTime, endTime, projectId,- parentObservationId, type-- - `basic` - name, level, statusMessage, version, environment,- bookmarked, public, userId, sessionId, isRootObservation-- - `time` - completionStartTime, createdAt, updatedAt-- - `io` - input, output-- - `metadata` - metadata (truncated to 200 chars by default, use- `expandMetadata` to get full values)-- - `model` - providedModelName, internalModelId, modelParameters-- - `usage` - usageDetails, costDetails, totalCost, usagePricingTierName-- - `prompt` - promptId, promptName, promptVersion-- - `metrics` - latency, timeToFirstToken-- - `trace_context` - tags, release, traceName--- If not specified, `core` and `basic` field groups are returned.--- ## Filters-- Multiple filtering options are available via query parameters or the- structured `filter` parameter.-- When using the `filter` parameter, it takes precedence over individual- query parameter filters.- operationId: observations_getMany- tags:- - Observations- parameters:- - name: fields- in: query- description: >-- Comma-separated list of field groups to include in the response.-- Available groups: core, basic, time, io, metadata, model, usage,- prompt, metrics, trace_context.-- If not specified, `core` and `basic` field groups are returned.-- Example: "basic,usage,model"- required: false- schema:- type: string- nullable: true- - name: expandMetadata- in: query- description: |-- Comma-separated list of metadata keys to return non-truncated.- By default, metadata values over 200 characters are truncated.- Use this parameter to retrieve full values for specific keys.- Example: "key1,key2"- required: false- schema:- type: string- nullable: true- - name: limit- in: query- description: Number of items to return per page. Maximum 1000, default 50.- required: false- schema:- type: integer- nullable: true- - name: cursor- in: query- description: >-- Base64-encoded cursor for pagination. Use the cursor from the- previous response to get the next page.- required: false- schema:- type: string- nullable: true- - name: parseIoAsJson- in: query- description: |-- **Deprecated.** Setting this to `true` will return a 400 error.- Input/output fields are always returned as raw strings.- Remove this parameter or set it to `false`.- required: false- schema:- type: boolean- nullable: true- - name: name- in: query- required: false- schema:- type: string- nullable: true- - name: userId- in: query- required: false- schema:- type: string- nullable: true- - name: sessionId- in: query- description: Filter by session ID.- required: false+ nullable: true+ - name: userId+ in: query+ required: falseschema:type: stringnullable: true- name: typein: query- description: >-- Filter by observation type (e.g., "GENERATION", "SPAN", "EVENT",- "AGENT", "TOOL", "CHAIN", "RETRIEVER", "EVALUATOR", "EMBEDDING",- "GUARDRAIL")required: falseschema:type: string@@ -3587,30 +3117,10 @@ paths:nullable: true- name: parentObservationIdin: query- description: >-- Filter by the physical parent observation ID.-- An empty value matches only observations without a physical parent.- Use `isRootObservation` to include observations marked as app roots- by the SDK, which may retain a non-null `parentObservationId`.required: falseschema:type: stringnullable: true- - name: isRootObservation- in: query- description: >-- Filter by whether an observation is a logical root.-- Root observations include observations without a physical parent and- observations marked as app roots by the SDK.-- An app-root observation may have `isRootObservation=true` and a- non-null `parentObservationId`.- required: false- schema:- type: boolean- nullable: true- name: environmentin: querydescription: >-@@ -3669,12 +3179,12 @@ paths:"column": string, // Required. Column to filter on (see available columns below)"operator": string, // Required. Operator based on type:// - datetime: ">", "<", ">=", "<="- // - string: "=", "contains", "does not contain", "starts with", "ends with", "matches"+ // - string: "=", "contains", "does not contain", "starts with", "ends with"// - stringOptions: "any of", "none of"// - categoryOptions: "any of", "none of"// - arrayOptions: "any of", "none of", "all of"// - number: "=", ">", "<", ">=", "<="- // - stringObject: "=", "contains", "does not contain", "starts with", "ends with", "matches"+ // - stringObject: "=", "contains", "does not contain", "starts with", "ends with"// - numberObject: "=", ">", "<", ">=", "<="// - boolean: "=", "<>"// - null: "is null", "is not null"@@ -3711,23 +3221,6 @@ paths:- `version` (string) - Version tag- - `userId` (string) - User ID-- - `sessionId` (string) - Session ID-- - `isRootObservation` (boolean) - Whether the observation is a- logical root. Observations marked as app roots by the SDK may retain- a non-null parentObservationId.--- ### Trace-Related Fields-- - `traceName` (string) - Name of the parent trace-- - `traceTags` (arrayOptions) - Tags from the parent trace-- - `tags` (arrayOptions) - Alias for traceTags-### Performance Metrics@@ -3759,8 +3252,7 @@ paths:### Model Information- - `model` (string) - Provided model name (alias:- `providedModelName`)+ - `model` (string) - Provided model name- `promptName` (string) - Associated prompt name@@ -3769,28 +3261,20 @@ paths:### Structured Data- - `input` (string) - Observation input. Supports accelerated indexed- literal search with the `matches` operator.-- - `output` (string) - Observation output. Supports accelerated- indexed literal search with the `matches` operator.-- `metadata` (stringObject/numberObject/categoryOptions) - Metadatakey-value pairs. Use `key` parameter to filter on specific metadatakeys.- The `matches` operator is only supported for `input`, `output`, and- stringObject `metadata` filters. It performs indexed literal search- with token-boundary pruning using the events table text indexes.- Case sensitivity differs by target: `input` and `output` matches are- case-insensitive, while metadata value matches are case-sensitive.- Unlike SQL `LIKE`, `%` and `_` are treated as literal characters.- Use `contains` for legacy substring semantics where the API allows- it. Any v2 `input` or `output` filter must be accompanied by at- least one `=` or `matches` filter on `input` or `output`; standalone- `contains`, `starts with`, `ends with`, and `does not contain`- filters on these columns are rejected.+ ### Associated Trace Fields (requires join with traces table)++ - `userId` (string) - User ID from associated trace++ - `traceName` (string) - Name from associated trace++ - `traceEnvironment` (string) - Environment from associated trace++ - `traceTags` (arrayOptions) - Tags from associated trace## Filter Examples@@ -3816,18 +3300,6 @@ paths:"key": "environment","operator": "=","value": "production"- },- {- "type": "string",- "column": "output",- "operator": "matches",- "value": "needle"- },- {- "type": "boolean",- "column": "isRootObservation",- "operator": "=",- "value": true}]@@ -3842,7 +3314,7 @@ paths:content:application/json:schema:- $ref: '#/components/schemas/ObservationsV2Response'+ $ref: '#/components/schemas/legacyObservationsViews''400':description: ''content:@@ -3869,51 +3341,23 @@ paths:application/json:schema: {}security: *ref_0- /api/public/otel/v1/traces:- post:- description: >-- **OpenTelemetry Traces Ingestion Endpoint**--- This endpoint implements the OTLP/HTTP specification for trace- ingestion, providing native OpenTelemetry integration for Langfuse- Observability.--- **Supported Formats:**-- - Binary Protobuf: `Content-Type: application/x-protobuf`-- - JSON Protobuf: `Content-Type: application/json`-- - Supports gzip compression via `Content-Encoding: gzip` header--- **Specification Compliance:**-- - Conforms to [OTLP/HTTP Trace- Export](https://opentelemetry.io/docs/specs/otlp/#otlphttp)-- - Implements `ExportTraceServiceRequest` message format--- **Documentation:**-- - Integration guide:- https://langfuse.com/integrations/native/opentelemetry-- - Data model: https://langfuse.com/docs/observability/data-model- operationId: opentelemetry_exportTraces+ deprecated: true+ /api/public/scores/{scoreId}:+ delete:+ description: Delete a score (supports both trace and session scores)+ operationId: legacy_scoreV1_deletetags:- - Opentelemetry- parameters: []+ - LegacyScoreV1+ parameters:+ - name: scoreId+ in: path+ description: The unique langfuse identifier of a score+ required: true+ schema:+ type: stringresponses:- '200':+ '204':description: ''- content:- application/json:- schema:- $ref: '#/components/schemas/OtelTraceResponse''400':description: ''content:@@ -3940,38 +3384,34 @@ paths:application/json:schema: {}security: *ref_0- requestBody:- required: true- content:- application/json:- schema:- type: object- properties:- resourceSpans:- type: array- items:- $ref: '#/components/schemas/OtelResourceSpan'- description: >-- Array of resource spans containing trace data as defined in- the OTLP specification- required:- - resourceSpans- /api/public/organizations/memberships:+ /api/public/llm-connections:get:- description: >-- Get all memberships for the organization associated with the API key- (requires organization-scoped API key)- operationId: organizations_getOrganizationMemberships+ description: Get all LLM connections in a project+ operationId: llmConnections_listtags:- - Organizations- parameters: []+ - LlmConnections+ parameters:+ - name: page+ in: query+ description: page number, starts at 1+ required: false+ schema:+ type: integer+ nullable: true+ - name: limit+ in: query+ description: limit of items per page+ required: false+ schema:+ type: integer+ nullable: trueresponses:'200':description: ''content:application/json:schema:- $ref: '#/components/schemas/MembershipsResponse'+ $ref: '#/components/schemas/PaginatedLlmConnections''400':description: ''content:@@ -4000,11 +3440,11 @@ paths:security: *ref_0put:description: >-- Create or update a membership for the organization associated with the- API key (requires organization-scoped API key)- operationId: organizations_updateOrganizationMembership+ Create or update an LLM connection. The connection is upserted on+ provider.+ operationId: llmConnections_upserttags:- - Organizations+ - LlmConnectionsparameters: []responses:'200':@@ -4012,7 +3452,7 @@ paths:content:application/json:schema:- $ref: '#/components/schemas/MembershipResponse'+ $ref: '#/components/schemas/LlmConnection''400':description: ''content:@@ -4044,64 +3484,17 @@ paths:content:application/json:schema:- $ref: '#/components/schemas/MembershipRequest'+ $ref: '#/components/schemas/UpsertLlmConnectionRequest'+ /api/public/llm-connections/{id}:delete:description: >-- Delete a membership from the organization associated with the API key- (requires organization-scoped API key)- operationId: organizations_deleteOrganizationMembership- tags:- - Organizations- parameters: []- responses:- '200':- description: ''- content:- application/json:- schema:- $ref: '#/components/schemas/MembershipDeletionResponse'- '400':- description: ''- content:- application/json:- schema: {}- '401':- description: ''- content:- application/json:- schema: {}- '403':- description: ''- content:- application/json:- schema: {}- '404':- description: ''- content:- application/json:- schema: {}- '405':- description: ''- content:- application/json:- schema: {}- security: *ref_0- requestBody:- required: true- content:- application/json:- schema:- $ref: '#/components/schemas/DeleteMembershipRequest'- /api/public/projects/{projectId}/memberships:- get:- description: >-- Get all memberships for a specific project (requires organization-scoped- API key)- operationId: organizations_getProjectMemberships+ Delete an LLM connection by id. Evaluators that depend on the deleted+ connection are automatically paused.+ operationId: llmConnections_deletetags:- - Organizations+ - LlmConnectionsparameters:- - name: projectId+ - name: idin: pathrequired: trueschema:@@ -4112,7 +3505,7 @@ paths:content:application/json:schema:- $ref: '#/components/schemas/MembershipsResponse'+ $ref: '#/components/schemas/DeleteLlmConnectionResponse''400':description: ''content:@@ -4139,17 +3532,16 @@ paths:application/json:schema: {}security: *ref_0- put:- description: >-- Create or update a membership for a specific project (requires- organization-scoped API key). The user must already be a member of the- organization.- operationId: organizations_updateProjectMembership+ /api/public/media/{mediaId}:+ get:+ description: Get a media record+ operationId: media_gettags:- - Organizations+ - Mediaparameters:- - name: projectId+ - name: mediaIdin: path+ description: The unique langfuse identifier of a media recordrequired: trueschema:type: string@@ -4159,7 +3551,7 @@ paths:content:application/json:schema:- $ref: '#/components/schemas/MembershipResponse'+ $ref: '#/components/schemas/GetMediaResponse''400':description: ''content:@@ -4186,33 +3578,21 @@ paths:application/json:schema: {}security: *ref_0- requestBody:- required: true- content:- application/json:- schema:- $ref: '#/components/schemas/MembershipRequest'- delete:- description: >-- Delete a membership from a specific project (requires- organization-scoped API key). The user must be a member of the- organization.- operationId: organizations_deleteProjectMembership+ patch:+ description: Patch a media record+ operationId: media_patchtags:- - Organizations+ - Mediaparameters:- - name: projectId+ - name: mediaIdin: path+ description: The unique langfuse identifier of a media recordrequired: trueschema:type: stringresponses:- '200':+ '204':description: ''- content:- application/json:- schema:- $ref: '#/components/schemas/MembershipDeletionResponse''400':description: ''content:@@ -4244,15 +3624,13 @@ paths:content:application/json:schema:- $ref: '#/components/schemas/DeleteMembershipRequest'- /api/public/organizations/projects:- get:- description: >-- Get all projects for the organization associated with the API key- (requires organization-scoped API key)- operationId: organizations_getOrganizationProjects+ $ref: '#/components/schemas/PatchMediaBody'+ /api/public/media:+ post:+ description: Get a presigned upload URL for a media record+ operationId: media_getUploadUrltags:- - Organizations+ - Mediaparameters: []responses:'200':@@ -4260,7 +3638,7 @@ paths:content:application/json:schema:- $ref: '#/components/schemas/OrganizationProjectsResponse'+ $ref: '#/components/schemas/GetMediaUploadUrlResponse''400':description: ''content:@@ -4287,24 +3665,1467 @@ paths:application/json:schema: {}security: *ref_0- /api/public/organizations/apiKeys:+ requestBody:+ required: true+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/GetMediaUploadUrlRequest'+ /api/public/v2/metrics:get:description: >-- Get all API keys for the organization associated with the API key- (requires organization-scoped API key)- operationId: organizations_getOrganizationApiKeys- tags:- - Organizations- parameters: []- responses:- '200':- description: ''- content:- application/json:- schema:- $ref: '#/components/schemas/OrganizationApiKeysResponse'- '400':- description: ''+ Get metrics from the Langfuse project using a query object. V2 endpoint+ with optimized performance.+++ ## V2 Differences++ - Supports `observations`, `scores-numeric`, `scores-boolean`, and+ `scores-categorical` views only (traces view not supported)++ - Direct access to tags and release fields on observations++ - Semantic-root filtering and grouping through the v2-only+ `isRootObservation` dimension++ - Backwards-compatible: traceName, traceRelease, traceVersion dimensions+ are still available on observations view++ - High cardinality dimensions are not supported and will return a 400+ error (see below)+++ For more details, see the [Metrics API+ documentation](https://langfuse.com/docs/metrics/features/metrics-api).+++ ## Available Views+++ ### observations++ Query observation-level data (spans, generations, events).+++ **Dimensions:**++ - `environment` - Deployment environment (e.g., production, staging)++ - `type` - Type of observation (SPAN, GENERATION, EVENT)++ - `name` - Name of the observation++ - `level` - Logging level of the observation++ - `version` - Version of the observation++ - `tags` - User-defined tags++ - `release` - Release version++ - `traceName` - Name of the parent trace (backwards-compatible)++ - `traceRelease` - Release version of the parent trace+ (backwards-compatible, maps to release)++ - `traceVersion` - Version of the parent trace (backwards-compatible,+ maps to version)++ - `providedModelName` - Name of the model used++ - `promptName` - Name of the prompt used++ - `promptVersion` - Version of the prompt used++ - `isRootObservation` - Boolean semantic-root status. `true` includes+ physical roots and app roots whose SDK parent is external (so+ `parentObservationId` may be non-null).++ - `startTimeMonth` - Month of start_time in YYYY-MM format+++ **Measures:**++ - `count` - Total number of observations++ - `latency` - Observation latency (milliseconds)++ - `streamingLatency` - Generation latency from completion start to end+ (milliseconds)++ - `inputTokens` - Sum of input tokens consumed++ - `outputTokens` - Sum of output tokens produced++ - `totalTokens` - Sum of all tokens consumed++ - `outputTokensPerSecond` - Output tokens per second++ - `tokensPerSecond` - Total tokens per second++ - `inputCost` - Input cost (USD)++ - `outputCost` - Output cost (USD)++ - `totalCost` - Total cost (USD)++ - `timeToFirstToken` - Time to first token (milliseconds)++ - `countScores` - Number of scores attached to the observation+++ ### scores-numeric++ Query numeric and boolean score data.+++ **Dimensions:**++ - `environment` - Deployment environment++ - `name` - Name of the score (e.g., accuracy, toxicity)++ - `source` - Origin of the score (API, ANNOTATION, EVAL)++ - `dataType` - Data type (NUMERIC, BOOLEAN)++ - `configId` - Identifier of the score config++ - `timestampMonth` - Month in YYYY-MM format++ - `timestampDay` - Day in YYYY-MM-DD format++ - `value` - Numeric value of the score++ - `traceName` - Name of the parent trace++ - `tags` - Tags++ - `traceRelease` - Release version++ - `traceVersion` - Version++ - `observationName` - Name of the associated observation++ - `observationModelName` - Model name of the associated observation++ - `observationPromptName` - Prompt name of the associated observation++ - `observationPromptVersion` - Prompt version of the associated+ observation+++ **Measures:**++ - `count` - Total number of scores++ - `value` - Score value (for aggregations)+++ ### scores-boolean++ Query boolean score data. It has the same score and parent+ trace/observation dimensions as scores-numeric, plus:+++ **Dimensions:**++ - `booleanValue` - Boolean value for true/false grouping and filtering+++ **Measures:**++ - `count` - Total number of boolean scores++ - `value` - Numeric 0/1 score value; `avg` returns the true-rate+++ ### scores-categorical++ Query categorical score data. Same dimensions as scores-numeric except+ uses `stringValue` instead of `value`.+++ **Measures:**++ - `count` - Total number of scores+++ ## High Cardinality Dimensions++ The following dimensions cannot be used as grouping dimensions in v2+ metrics API as they can cause performance issues.++ Use them in filters instead.+++ **observations view:**++ - `id` - Use traceId filter to narrow down results++ - `traceId` - Use traceId filter instead++ - `userId` - Use userId filter instead++ - `sessionId` - Use sessionId filter instead++ - `parentObservationId` - Use parentObservationId filter instead+++ **scores-numeric / scores-boolean / scores-categorical views:**++ - `id` - Use specific filters to narrow down results++ - `traceId` - Use traceId filter instead++ - `userId` - Use userId filter instead++ - `sessionId` - Use sessionId filter instead++ - `observationId` - Use observationId filter instead+++ ## Aggregations++ Available aggregation functions: `sum`, `avg`, `count`, `max`, `min`,+ `p50`, `p75`, `p90`, `p95`, `p99`, `histogram`+++ ## Time Granularities++ Available granularities for timeDimension: `auto`, `minute`, `hour`,+ `day`, `week`, `month`++ - `auto` bins the data into approximately 50 buckets based on the time+ range+ operationId: metrics_metrics+ tags:+ - Metrics+ parameters:+ - name: query+ in: query+ description: >-+ JSON string containing the query parameters with the following+ structure:++ ```json++ {+ "view": string, // Required. One of "observations", "scores-numeric", "scores-boolean", "scores-categorical"+ "dimensions": [ // Optional. Default: []+ {+ "field": string // Field to group by (see available dimensions above)+ }+ ],+ "metrics": [ // Required. At least one metric must be provided+ {+ "measure": string, // What to measure (see available measures above)+ "aggregation": string // How to aggregate: "sum", "avg", "count", "max", "min", "p50", "p75", "p90", "p95", "p99", "histogram"+ }+ ],+ "filters": [ // Optional. Default: []+ {+ "column": string, // Column to filter on (any dimension field)+ "operator": string, // Operator based on type:+ // - datetime: ">", "<", ">=", "<="+ // - string: "=", "contains", "does not contain", "starts with", "ends with"+ // - stringOptions: "any of", "none of"+ // - arrayOptions: "any of", "none of", "all of"+ // - number: "=", ">", "<", ">=", "<="+ // - stringObject/numberObject: same as string/number with required "key"+ // - boolean: "=", "<>"+ // - null: "is null", "is not null"+ "value": any, // Value to compare against+ "type": string, // Data type: "datetime", "string", "number", "stringOptions", "categoryOptions", "arrayOptions", "stringObject", "numberObject", "boolean", "null"+ "key": string // Required only for stringObject/numberObject types (e.g., metadata filtering)+ }+ ],+ "timeDimension": { // Optional. Default: null. If provided, results will be grouped by time+ "granularity": string // One of "auto", "minute", "hour", "day", "week", "month"+ },+ "fromTimestamp": string, // Required. ISO datetime string for start of time range+ "toTimestamp": string, // Required. ISO datetime string for end of time range (must be after fromTimestamp)+ "orderBy": [ // Optional. Default: null+ {+ "field": string, // Field to order by (dimension or metric alias)+ "direction": string // "asc" or "desc"+ }+ ],+ "config": { // Optional. Query-specific configuration+ "bins": number, // Optional. Number of bins for histogram aggregation (1-100), default: 10+ "row_limit": number // Optional. Maximum number of rows to return (1-1000), default: 100+ }+ }++ ```+++ For example, to count semantic roots (including app roots with a+ non-null external parent), use a boolean filter:++ ```json++ {+ "view": "observations",+ "metrics": [{"measure": "count", "aggregation": "count"}],+ "filters": [{"column": "isRootObservation", "operator": "=", "value": true, "type": "boolean"}],+ "fromTimestamp": "2025-01-01T00:00:00.000Z",+ "toTimestamp": "2025-02-01T00:00:00.000Z"+ }++ ```+ required: true+ schema:+ type: string+ responses:+ '200':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/MetricsV2Response'+ '400':+ description: ''+ content:+ application/json:+ schema: {}+ '401':+ description: ''+ content:+ application/json:+ schema: {}+ '403':+ description: ''+ content:+ application/json:+ schema: {}+ '404':+ description: ''+ content:+ application/json:+ schema: {}+ '405':+ description: ''+ content:+ application/json:+ schema: {}+ security: *ref_0+ /api/public/models:+ post:+ description: Create a model+ operationId: models_create+ tags:+ - Models+ parameters: []+ responses:+ '200':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/Model'+ '400':+ description: ''+ content:+ application/json:+ schema: {}+ '401':+ description: ''+ content:+ application/json:+ schema: {}+ '403':+ description: ''+ content:+ application/json:+ schema: {}+ '404':+ description: ''+ content:+ application/json:+ schema: {}+ '405':+ description: ''+ content:+ application/json:+ schema: {}+ security: *ref_0+ requestBody:+ required: true+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/CreateModelRequest'+ get:+ description: Get all models+ operationId: models_list+ tags:+ - Models+ parameters:+ - name: page+ in: query+ description: page number, starts at 1+ required: false+ schema:+ type: integer+ nullable: true+ - name: limit+ in: query+ description: limit of items per page+ required: false+ schema:+ type: integer+ nullable: true+ responses:+ '200':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/PaginatedModels'+ '400':+ description: ''+ content:+ application/json:+ schema: {}+ '401':+ description: ''+ content:+ application/json:+ schema: {}+ '403':+ description: ''+ content:+ application/json:+ schema: {}+ '404':+ description: ''+ content:+ application/json:+ schema: {}+ '405':+ description: ''+ content:+ application/json:+ schema: {}+ security: *ref_0+ /api/public/models/{id}:+ put:+ description: >-+ Create or replace a project-owned model using its id. Built-in models+ cannot be modified.+ operationId: models_upsert+ tags:+ - Models+ parameters:+ - name: id+ in: path+ required: true+ schema:+ type: string+ responses:+ '200':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/Model'+ '400':+ description: ''+ content:+ application/json:+ schema: {}+ '401':+ description: ''+ content:+ application/json:+ schema: {}+ '403':+ description: ''+ content:+ application/json:+ schema: {}+ '404':+ description: ''+ content:+ application/json:+ schema: {}+ '405':+ description: ''+ content:+ application/json:+ schema: {}+ security: *ref_0+ requestBody:+ required: true+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/CreateModelRequest'+ get:+ description: Get a model+ operationId: models_get+ tags:+ - Models+ parameters:+ - name: id+ in: path+ required: true+ schema:+ type: string+ responses:+ '200':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/Model'+ '400':+ description: ''+ content:+ application/json:+ schema: {}+ '401':+ description: ''+ content:+ application/json:+ schema: {}+ '403':+ description: ''+ content:+ application/json:+ schema: {}+ '404':+ description: ''+ content:+ application/json:+ schema: {}+ '405':+ description: ''+ content:+ application/json:+ schema: {}+ security: *ref_0+ delete:+ description: >-+ Delete a model. Cannot delete models managed by Langfuse. You can create+ your own definition with the same modelName to override the definition+ though.+ operationId: models_delete+ tags:+ - Models+ parameters:+ - name: id+ in: path+ required: true+ schema:+ type: string+ responses:+ '204':+ description: ''+ '400':+ description: ''+ content:+ application/json:+ schema: {}+ '401':+ description: ''+ content:+ application/json:+ schema: {}+ '403':+ description: ''+ content:+ application/json:+ schema: {}+ '404':+ description: ''+ content:+ application/json:+ schema: {}+ '405':+ description: ''+ content:+ application/json:+ schema: {}+ security: *ref_0+ /api/public/v2/observations:+ get:+ description: >-+ Get a list of observations with cursor-based pagination and flexible+ field selection.+++ ## Cursor-based Pagination++ This endpoint uses cursor-based pagination for efficient traversal of+ large datasets.++ The cursor is returned in the response metadata and should be passed in+ subsequent requests++ to retrieve the next page of results.+++ ## Field Selection++ Use the `fields` parameter to control which observation fields are+ returned:++ - `core` - Always included: id, traceId, startTime, endTime, projectId,+ parentObservationId, type++ - `basic` - name, level, statusMessage, version, environment,+ bookmarked, public, userId, sessionId, isRootObservation++ - `time` - completionStartTime, createdAt, updatedAt++ - `io` - input, output++ - `metadata` - metadata (truncated to 200 chars by default, use+ `expandMetadata` to get full values)++ - `model` - providedModelName, internalModelId, modelParameters++ - `usage` - usageDetails, costDetails, totalCost, usagePricingTierName++ - `prompt` - promptId, promptName, promptVersion++ - `metrics` - latency, timeToFirstToken++ - `trace_context` - tags, release, traceName+++ If not specified, `core` and `basic` field groups are returned.+++ ## Filters++ Multiple filtering options are available via query parameters or the+ structured `filter` parameter.++ When using the `filter` parameter, it takes precedence over individual+ query parameter filters.+ operationId: observations_getMany+ tags:+ - Observations+ parameters:+ - name: fields+ in: query+ description: >-+ Comma-separated list of field groups to include in the response.++ Available groups: core, basic, time, io, metadata, model, usage,+ prompt, metrics, trace_context.++ If not specified, `core` and `basic` field groups are returned.++ Example: "basic,usage,model"+ required: false+ schema:+ type: string+ nullable: true+ - name: expandMetadata+ in: query+ description: |-+ Comma-separated list of metadata keys to return non-truncated.+ By default, metadata values over 200 characters are truncated.+ Use this parameter to retrieve full values for specific keys.+ Example: "key1,key2"+ required: false+ schema:+ type: string+ nullable: true+ - name: limit+ in: query+ description: Number of items to return per page. Maximum 1000, default 50.+ required: false+ schema:+ type: integer+ nullable: true+ - name: cursor+ in: query+ description: >-+ Base64-encoded cursor for pagination. Use the cursor from the+ previous response to get the next page.+ required: false+ schema:+ type: string+ nullable: true+ - name: parseIoAsJson+ in: query+ description: |-+ **Deprecated.** Setting this to `true` will return a 400 error.+ Input/output fields are always returned as raw strings.+ Remove this parameter or set it to `false`.+ required: false+ schema:+ type: boolean+ nullable: true+ - name: name+ in: query+ required: false+ schema:+ type: string+ nullable: true+ - name: userId+ in: query+ required: false+ schema:+ type: string+ nullable: true+ - name: sessionId+ in: query+ description: Filter by session ID.+ required: false+ schema:+ type: string+ nullable: true+ - name: type+ in: query+ description: >-+ Filter by observation type (e.g., "GENERATION", "SPAN", "EVENT",+ "AGENT", "TOOL", "CHAIN", "RETRIEVER", "EVALUATOR", "EMBEDDING",+ "GUARDRAIL")+ required: false+ schema:+ type: string+ nullable: true+ - name: traceId+ in: query+ required: false+ schema:+ type: string+ nullable: true+ - name: level+ in: query+ description: >-+ Optional filter for observations with a specific level (e.g.+ "DEBUG", "DEFAULT", "WARNING", "ERROR").+ required: false+ schema:+ $ref: '#/components/schemas/ObservationLevel'+ nullable: true+ - name: parentObservationId+ in: query+ description: >-+ Filter by the physical parent observation ID.++ An empty value matches only observations without a physical parent.+ Use `isRootObservation` to include observations marked as app roots+ by the SDK, which may retain a non-null `parentObservationId`.+ required: false+ schema:+ type: string+ nullable: true+ - name: isRootObservation+ in: query+ description: >-+ Filter by whether an observation is a logical root.++ Root observations include observations without a physical parent and+ observations marked as app roots by the SDK.++ An app-root observation may have `isRootObservation=true` and a+ non-null `parentObservationId`.+ required: false+ schema:+ type: boolean+ nullable: true+ - name: environment+ in: query+ description: >-+ Optional filter for observations where the environment is one of the+ provided values.+ required: false+ schema:+ type: array+ items:+ type: string+ nullable: true+ - name: fromStartTime+ in: query+ description: >-+ Retrieve only observations with a start_time on or after this+ datetime (ISO 8601).+ required: false+ schema:+ type: string+ format: date-time+ nullable: true+ - name: toStartTime+ in: query+ description: >-+ Retrieve only observations with a start_time before this datetime+ (ISO 8601).+ required: false+ schema:+ type: string+ format: date-time+ nullable: true+ - name: version+ in: query+ description: Optional filter to only include observations with a certain version.+ required: false+ schema:+ type: string+ nullable: true+ - name: filter+ in: query+ description: >-+ JSON string containing an array of filter conditions. When provided,+ this takes precedence over query parameter filters (userId, name,+ type, level, environment, fromStartTime, ...).+++ ## Filter Structure++ Each filter condition has the following structure:++ ```json++ [+ {+ "type": string, // Required. One of: "datetime", "string", "number", "stringOptions", "categoryOptions", "arrayOptions", "stringObject", "numberObject", "boolean", "null"+ "column": string, // Required. Column to filter on (see available columns below)+ "operator": string, // Required. Operator based on type:+ // - datetime: ">", "<", ">=", "<="+ // - string: "=", "contains", "does not contain", "starts with", "ends with", "matches"+ // - stringOptions: "any of", "none of"+ // - categoryOptions: "any of", "none of"+ // - arrayOptions: "any of", "none of", "all of"+ // - number: "=", ">", "<", ">=", "<="+ // - stringObject: "=", "contains", "does not contain", "starts with", "ends with", "matches"+ // - numberObject: "=", ">", "<", ">=", "<="+ // - boolean: "=", "<>"+ // - null: "is null", "is not null"+ "value": any, // Required (except for null type). Value to compare against. Type depends on filter type+ "key": string // Required only for stringObject, numberObject, and categoryOptions types when filtering on nested fields like metadata+ }+ ]++ ```+++ ## Available Columns+++ ### Core Observation Fields++ - `id` (string) - Observation ID++ - `type` (string) - Observation type (SPAN, GENERATION, EVENT)++ - `name` (string) - Observation name++ - `traceId` (string) - Associated trace ID++ - `startTime` (datetime) - Observation start time++ - `endTime` (datetime) - Observation end time++ - `environment` (string) - Environment tag++ - `level` (string) - Log level (DEBUG, DEFAULT, WARNING, ERROR)++ - `statusMessage` (string) - Status message++ - `version` (string) - Version tag++ - `userId` (string) - User ID++ - `sessionId` (string) - Session ID++ - `isRootObservation` (boolean) - Whether the observation is a+ logical root. Observations marked as app roots by the SDK may retain+ a non-null parentObservationId.+++ ### Trace-Related Fields++ - `traceName` (string) - Name of the parent trace++ - `traceTags` (arrayOptions) - Tags from the parent trace++ - `tags` (arrayOptions) - Alias for traceTags+++ ### Performance Metrics++ - `latency` (number) - Latency in seconds (calculated: end_time -+ start_time)++ - `timeToFirstToken` (number) - Time to first token in seconds++ - `tokensPerSecond` (number) - Output tokens per second+++ ### Token Usage++ - `inputTokens` (number) - Number of input tokens++ - `outputTokens` (number) - Number of output tokens++ - `totalTokens` (number) - Total tokens (alias: `tokens`)+++ ### Cost Metrics++ - `inputCost` (number) - Input cost in USD++ - `outputCost` (number) - Output cost in USD++ - `totalCost` (number) - Total cost in USD+++ ### Model Information++ - `model` (string) - Provided model name (alias:+ `providedModelName`)++ - `promptName` (string) - Associated prompt name++ - `promptVersion` (number) - Associated prompt version+++ ### Structured Data++ - `input` (string) - Observation input. Supports accelerated indexed+ literal search with the `matches` operator.++ - `output` (string) - Observation output. Supports accelerated+ indexed literal search with the `matches` operator.++ - `metadata` (stringObject/numberObject/categoryOptions) - Metadata+ key-value pairs. Use `key` parameter to filter on specific metadata+ keys.+++ The `matches` operator is only supported for `input`, `output`, and+ stringObject `metadata` filters. It performs indexed literal search+ with token-boundary pruning using the events table text indexes.+ Case sensitivity differs by target: `input` and `output` matches are+ case-insensitive, while metadata value matches are case-sensitive.+ Unlike SQL `LIKE`, `%` and `_` are treated as literal characters.+ Use `contains` for legacy substring semantics where the API allows+ it. Any v2 `input` or `output` filter must be accompanied by at+ least one `=` or `matches` filter on `input` or `output`; standalone+ `contains`, `starts with`, `ends with`, and `does not contain`+ filters on these columns are rejected.+++ ## Filter Examples++ ```json++ [+ {+ "type": "string",+ "column": "type",+ "operator": "=",+ "value": "GENERATION"+ },+ {+ "type": "number",+ "column": "latency",+ "operator": ">=",+ "value": 2.5+ },+ {+ "type": "stringObject",+ "column": "metadata",+ "key": "environment",+ "operator": "=",+ "value": "production"+ },+ {+ "type": "string",+ "column": "output",+ "operator": "matches",+ "value": "needle"+ },+ {+ "type": "boolean",+ "column": "isRootObservation",+ "operator": "=",+ "value": true+ }+ ]++ ```+ required: false+ schema:+ type: string+ nullable: true+ responses:+ '200':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/ObservationsV2Response'+ '400':+ description: ''+ content:+ application/json:+ schema: {}+ '401':+ description: ''+ content:+ application/json:+ schema: {}+ '403':+ description: ''+ content:+ application/json:+ schema: {}+ '404':+ description: ''+ content:+ application/json:+ schema: {}+ '405':+ description: ''+ content:+ application/json:+ schema: {}+ security: *ref_0+ /api/public/otel/v1/traces:+ post:+ description: >-+ **OpenTelemetry Traces Ingestion Endpoint**+++ This endpoint implements the OTLP/HTTP specification for trace+ ingestion, providing native OpenTelemetry integration for Langfuse+ Observability.+++ **Supported Formats:**++ - Binary Protobuf: `Content-Type: application/x-protobuf`++ - JSON Protobuf: `Content-Type: application/json`++ - Supports gzip compression via `Content-Encoding: gzip` header+++ **Specification Compliance:**++ - Conforms to [OTLP/HTTP Trace+ Export](https://opentelemetry.io/docs/specs/otlp/#otlphttp)++ - Implements `ExportTraceServiceRequest` message format+++ **Documentation:**++ - Integration guide:+ https://langfuse.com/integrations/native/opentelemetry++ - Data model: https://langfuse.com/docs/observability/data-model+ operationId: opentelemetry_exportTraces+ tags:+ - Opentelemetry+ parameters: []+ responses:+ '200':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/OtelTraceResponse'+ '400':+ description: ''+ content:+ application/json:+ schema: {}+ '401':+ description: ''+ content:+ application/json:+ schema: {}+ '403':+ description: ''+ content:+ application/json:+ schema: {}+ '404':+ description: ''+ content:+ application/json:+ schema: {}+ '405':+ description: ''+ content:+ application/json:+ schema: {}+ security: *ref_0+ requestBody:+ required: true+ content:+ application/json:+ schema:+ type: object+ properties:+ resourceSpans:+ type: array+ items:+ $ref: '#/components/schemas/OtelResourceSpan'+ description: >-+ Array of resource spans containing trace data as defined in+ the OTLP specification+ required:+ - resourceSpans+ /api/public/organizations/memberships:+ get:+ description: >-+ Get all memberships for the organization associated with the API key+ (requires organization-scoped API key)+ operationId: organizations_getOrganizationMemberships+ tags:+ - Organizations+ parameters: []+ responses:+ '200':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/MembershipsResponse'+ '400':+ description: ''+ content:+ application/json:+ schema: {}+ '401':+ description: ''+ content:+ application/json:+ schema: {}+ '403':+ description: ''+ content:+ application/json:+ schema: {}+ '404':+ description: ''+ content:+ application/json:+ schema: {}+ '405':+ description: ''+ content:+ application/json:+ schema: {}+ security: *ref_0+ put:+ description: >-+ Create or update a membership for the organization associated with the+ API key (requires organization-scoped API key)+ operationId: organizations_updateOrganizationMembership+ tags:+ - Organizations+ parameters: []+ responses:+ '200':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/MembershipResponse'+ '400':+ description: ''+ content:+ application/json:+ schema: {}+ '401':+ description: ''+ content:+ application/json:+ schema: {}+ '403':+ description: ''+ content:+ application/json:+ schema: {}+ '404':+ description: ''+ content:+ application/json:+ schema: {}+ '405':+ description: ''+ content:+ application/json:+ schema: {}+ security: *ref_0+ requestBody:+ required: true+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/MembershipRequest'+ delete:+ description: >-+ Delete a membership from the organization associated with the API key+ (requires organization-scoped API key)+ operationId: organizations_deleteOrganizationMembership+ tags:+ - Organizations+ parameters: []+ responses:+ '200':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/MembershipDeletionResponse'+ '400':+ description: ''+ content:+ application/json:+ schema: {}+ '401':+ description: ''+ content:+ application/json:+ schema: {}+ '403':+ description: ''+ content:+ application/json:+ schema: {}+ '404':+ description: ''+ content:+ application/json:+ schema: {}+ '405':+ description: ''+ content:+ application/json:+ schema: {}+ security: *ref_0+ requestBody:+ required: true+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/DeleteMembershipRequest'+ /api/public/projects/{projectId}/memberships:+ get:+ description: >-+ Get all memberships for a specific project (requires organization-scoped+ API key)+ operationId: organizations_getProjectMemberships+ tags:+ - Organizations+ parameters:+ - name: projectId+ in: path+ required: true+ schema:+ type: string+ responses:+ '200':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/MembershipsResponse'+ '400':+ description: ''+ content:+ application/json:+ schema: {}+ '401':+ description: ''+ content:+ application/json:+ schema: {}+ '403':+ description: ''+ content:+ application/json:+ schema: {}+ '404':+ description: ''+ content:+ application/json:+ schema: {}+ '405':+ description: ''+ content:+ application/json:+ schema: {}+ security: *ref_0+ put:+ description: >-+ Create or update a membership for a specific project (requires+ organization-scoped API key). The user must already be a member of the+ organization.+ operationId: organizations_updateProjectMembership+ tags:+ - Organizations+ parameters:+ - name: projectId+ in: path+ required: true+ schema:+ type: string+ responses:+ '200':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/MembershipResponse'+ '400':+ description: ''+ content:+ application/json:+ schema: {}+ '401':+ description: ''+ content:+ application/json:+ schema: {}+ '403':+ description: ''+ content:+ application/json:+ schema: {}+ '404':+ description: ''+ content:+ application/json:+ schema: {}+ '405':+ description: ''+ content:+ application/json:+ schema: {}+ security: *ref_0+ requestBody:+ required: true+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/MembershipRequest'+ delete:+ description: >-+ Delete a membership from a specific project (requires+ organization-scoped API key). The user must be a member of the+ organization.+ operationId: organizations_deleteProjectMembership+ tags:+ - Organizations+ parameters:+ - name: projectId+ in: path+ required: true+ schema:+ type: string+ responses:+ '200':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/MembershipDeletionResponse'+ '400':+ description: ''+ content:+ application/json:+ schema: {}+ '401':+ description: ''+ content:+ application/json:+ schema: {}+ '403':+ description: ''+ content:+ application/json:+ schema: {}+ '404':+ description: ''+ content:+ application/json:+ schema: {}+ '405':+ description: ''+ content:+ application/json:+ schema: {}+ security: *ref_0+ requestBody:+ required: true+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/DeleteMembershipRequest'+ /api/public/organizations/projects:+ get:+ description: >-+ Get all projects for the organization associated with the API key+ (requires organization-scoped API key)+ operationId: organizations_getOrganizationProjects+ tags:+ - Organizations+ parameters: []+ responses:+ '200':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/OrganizationProjectsResponse'+ '400':+ description: ''+ content:+ application/json:+ schema: {}+ '401':+ description: ''+ content:+ application/json:+ schema: {}+ '403':+ description: ''+ content:+ application/json:+ schema: {}+ '404':+ description: ''+ content:+ application/json:+ schema: {}+ '405':+ description: ''+ content:+ application/json:+ schema: {}+ security: *ref_0+ /api/public/organizations/apiKeys:+ get:+ description: >-+ Get all API keys for the organization associated with the API key+ (requires organization-scoped API key)+ operationId: organizations_getOrganizationApiKeys+ tags:+ - Organizations+ parameters: []+ responses:+ '200':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/OrganizationApiKeysResponse'+ '400':+ description: ''content:application/json:schema: {}@@ -6714,83 +7535,547 @@ paths:These fields aggregate observation levels within the trace:- - `level` (string) - Highest severity level (ERROR > WARNING >- DEFAULT > DEBUG)+ - `level` (string) - Highest severity level (ERROR > WARNING >+ DEFAULT > DEBUG)++ - `warningCount` (number) - Count of WARNING level observations++ - `errorCount` (number) - Count of ERROR level observations++ - `defaultCount` (number) - Count of DEFAULT level observations++ - `debugCount` (number) - Count of DEBUG level observations+++ ### Scores (requires join with scores table)++ - `scores_avg` (number) - Average of numeric scores (alias:+ `scores`)++ - `score_categories` (categoryOptions) - Categorical score values++ - `score_booleans` (booleanObject) - Boolean score values. Use `key`+ for the score name and a boolean `value`, e.g. `{"type":+ "booleanObject", "column": "score_booleans", "key": "is_correct",+ "operator": "=", "value": true}`. The `<>` operator also matches+ traces without a score of that name.+++ ## Filter Examples++ ```json++ [+ {+ "type": "datetime",+ "column": "timestamp",+ "operator": ">=",+ "value": "2024-01-01T00:00:00Z"+ },+ {+ "type": "string",+ "column": "userId",+ "operator": "=",+ "value": "user-123"+ },+ {+ "type": "number",+ "column": "totalCost",+ "operator": ">=",+ "value": 0.01+ },+ {+ "type": "arrayOptions",+ "column": "tags",+ "operator": "all of",+ "value": ["production", "critical"]+ },+ {+ "type": "stringObject",+ "column": "metadata",+ "key": "customer_tier",+ "operator": "=",+ "value": "enterprise"+ }+ ]++ ```+++ ## Performance Notes++ - Filtering on `userId`, `sessionId`, or `metadata` may enable skip+ indexes for better query performance++ - Score filters require a join with the scores table and may impact+ query performance+ required: false+ schema:+ type: string+ nullable: true+ responses:+ '200':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/Traces'+ '400':+ description: ''+ content:+ application/json:+ schema: {}+ '401':+ description: ''+ content:+ application/json:+ schema: {}+ '403':+ description: ''+ content:+ application/json:+ schema: {}+ '404':+ description: ''+ content:+ application/json:+ schema: {}+ '405':+ description: ''+ content:+ application/json:+ schema: {}+ security: *ref_0+ deprecated: true+ delete:+ description: Delete multiple traces+ operationId: trace_deleteMultiple+ tags:+ - Trace+ parameters: []+ responses:+ '200':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/DeleteTraceResponse'+ '400':+ description: ''+ content:+ application/json:+ schema: {}+ '401':+ description: ''+ content:+ application/json:+ schema: {}+ '403':+ description: ''+ content:+ application/json:+ schema: {}+ '404':+ description: ''+ content:+ application/json:+ schema: {}+ '405':+ description: ''+ content:+ application/json:+ schema: {}+ security: *ref_0+ requestBody:+ required: true+ content:+ application/json:+ schema:+ type: object+ properties:+ traceIds:+ type: array+ items:+ type: string+ description: List of trace IDs to delete+ required:+ - traceIds+ /api/public/unstable/dashboard-widgets:+ get:+ description: |-+ List dashboard widgets in the project, ordered by most recently+ updated first.++ Responses may include legacy `traces` widgets created before this+ API existed. New widgets cannot be created with `view: traces`.+ operationId: unstable_dashboardWidgets_list+ tags:+ - UnstableDashboardWidgets+ parameters:+ - name: page+ in: query+ description: 1-based page number. Defaults to `1`.+ required: false+ schema:+ type: integer+ nullable: true+ - name: limit+ in: query+ description: Maximum number of items per page. Defaults to `50`.+ required: false+ schema:+ type: integer+ nullable: true+ responses:+ '200':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/unstableDashboardWidgetList'+ '400':+ description: ''+ content:+ application/json:+ schema: {}+ '401':+ description: ''+ content:+ application/json:+ schema: {}+ '403':+ description: ''+ content:+ application/json:+ schema: {}+ '404':+ description: ''+ content:+ application/json:+ schema: {}+ '405':+ description: ''+ content:+ application/json:+ schema: {}+ '429':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/unstablePublicApiError'+ '500':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/unstablePublicApiError'+ security: *ref_0+ post:+ description: >-+ Create a dashboard widget (a standalone chart definition you place on++ any dashboard).- - `warningCount` (number) - Count of WARNING level observations- - `errorCount` (number) - Count of ERROR level observations+ This endpoint creates the widget only; place it on a dashboard via- - `defaultCount` (number) - Count of DEFAULT level observations+ `POST /dashboards/{dashboardId}/placements`.- - `debugCount` (number) - Count of DEBUG level observations+ Supported views are `observations`, `scores-numeric`, `scores-boolean`,+ and `scores-categorical`.- ### Scores (requires join with scores table)+ The legacy `traces` view is not supported by this unstable API.- - `scores_avg` (number) - Average of numeric scores (alias:- `scores`)+ Widgets are created as v2 internally.- - `score_categories` (categoryOptions) - Categorical score values- - `score_booleans` (booleanObject) - Boolean score values. Use `key`- for the score name and a boolean `value`, e.g. `{"type":- "booleanObject", "column": "score_booleans", "key": "is_correct",- "operator": "=", "value": true}`. The `<>` operator also matches- traces without a score of that name.+ `chartConfig` is optional and defaults to the plain config for+ `chartType`; when `chartConfig.type` is given it must match- ## Filter Examples+ `chartType`.- ```json- [- {- "type": "datetime",- "column": "timestamp",- "operator": ">=",- "value": "2024-01-01T00:00:00Z"- },- {- "type": "string",- "column": "userId",- "operator": "=",- "value": "user-123"- },- {- "type": "number",- "column": "totalCost",- "operator": ">=",- "value": 0.01- },- {- "type": "arrayOptions",- "column": "tags",- "operator": "all of",- "value": ["production", "critical"]- },- {- "type": "stringObject",- "column": "metadata",- "key": "customer_tier",- "operator": "=",- "value": "enterprise"- }- ]+ Unstable API note:- ```+ - This surface may evolve while dashboard/widget APIs are being+ finalized.+ operationId: unstable_dashboardWidgets_create+ tags:+ - UnstableDashboardWidgets+ parameters: []+ responses:+ '200':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/unstableDashboardWidget'+ '400':+ description: ''+ content:+ application/json:+ schema: {}+ '401':+ description: ''+ content:+ application/json:+ schema: {}+ '403':+ description: ''+ content:+ application/json:+ schema: {}+ '404':+ description: ''+ content:+ application/json:+ schema: {}+ '405':+ description: ''+ content:+ application/json:+ schema: {}+ '429':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/unstablePublicApiError'+ '500':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/unstablePublicApiError'+ security: *ref_0+ requestBody:+ required: true+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/unstableCreateDashboardWidgetRequest'+ /api/public/unstable/dashboard-widgets/{widgetId}:+ get:+ description: |-+ Get a dashboard widget by id.+ The response may use `view: traces` for legacy widgets.+ operationId: unstable_dashboardWidgets_get+ tags:+ - UnstableDashboardWidgets+ parameters:+ - name: widgetId+ in: path+ required: true+ schema:+ type: string+ responses:+ '200':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/unstableDashboardWidget'+ '400':+ description: ''+ content:+ application/json:+ schema: {}+ '401':+ description: ''+ content:+ application/json:+ schema: {}+ '403':+ description: ''+ content:+ application/json:+ schema: {}+ '404':+ description: ''+ content:+ application/json:+ schema: {}+ '405':+ description: ''+ content:+ application/json:+ schema: {}+ '429':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/unstablePublicApiError'+ '500':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/unstablePublicApiError'+ security: *ref_0+ patch:+ description: |-+ Update a dashboard widget.- ## Performance Notes+ All fields are optional; at least one field is required.+ Changing `chartType` without sending `chartConfig` resets the config+ to the new chart type's defaults. When `chartConfig.type` is given+ it must match the widget's (possibly updated) `chartType`.- - Filtering on `userId`, `sessionId`, or `metadata` may enable skip- indexes for better query performance+ `view` cannot be changed to the legacy `traces` value. Existing+ `traces` widgets may be updated on other fields.+ operationId: unstable_dashboardWidgets_update+ tags:+ - UnstableDashboardWidgets+ parameters:+ - name: widgetId+ in: path+ required: true+ schema:+ type: string+ responses:+ '200':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/unstableDashboardWidget'+ '400':+ description: ''+ content:+ application/json:+ schema: {}+ '401':+ description: ''+ content:+ application/json:+ schema: {}+ '403':+ description: ''+ content:+ application/json:+ schema: {}+ '404':+ description: ''+ content:+ application/json:+ schema: {}+ '405':+ description: ''+ content:+ application/json:+ schema: {}+ '429':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/unstablePublicApiError'+ '500':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/unstablePublicApiError'+ security: *ref_0+ requestBody:+ required: true+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/unstableUpdateDashboardWidgetRequest'+ delete:+ description: |-+ Delete a dashboard widget.- - Score filters require a join with the scores table and may impact- query performance- required: false+ The API returns `409` while the widget is still placed on a dashboard.+ Remove those placements first.+ operationId: unstable_dashboardWidgets_delete+ tags:+ - UnstableDashboardWidgets+ parameters:+ - name: widgetId+ in: path+ required: trueschema:type: string+ responses:+ '200':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/unstableDeleteDashboardWidgetResponse'+ '400':+ description: ''+ content:+ application/json:+ schema: {}+ '401':+ description: ''+ content:+ application/json:+ schema: {}+ '403':+ description: ''+ content:+ application/json:+ schema: {}+ '404':+ description: ''+ content:+ application/json:+ schema: {}+ '405':+ description: ''+ content:+ application/json:+ schema: {}+ '409':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/unstablePublicApiError'+ '429':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/unstablePublicApiError'+ '500':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/unstablePublicApiError'+ security: *ref_0+ /api/public/unstable/dashboards:+ get:+ description: |-+ List dashboards in the project, ordered by most recently updated+ first.+ operationId: unstable_dashboards_list+ tags:+ - UnstableDashboards+ parameters:+ - name: page+ in: query+ description: 1-based page number. Defaults to `1`.+ required: false+ schema:+ type: integer+ nullable: true+ - name: limit+ in: query+ description: Maximum number of items per page. Defaults to `50`.+ required: false+ schema:+ type: integernullable: trueresponses:'200':@@ -6798,7 +8083,7 @@ paths:content:application/json:schema:- $ref: '#/components/schemas/Traces'+ $ref: '#/components/schemas/unstableDashboardList''400':description: ''content:@@ -6824,13 +8109,24 @@ paths:content:application/json:schema: {}+ '429':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/unstablePublicApiError'+ '500':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/unstablePublicApiError'security: *ref_0- deprecated: true- delete:- description: Delete multiple traces- operationId: trace_deleteMultiple+ post:+ description: Create a dashboard.+ operationId: unstable_dashboards_createtags:- - Trace+ - UnstableDashboardsparameters: []responses:'200':@@ -6838,7 +8134,7 @@ paths:content:application/json:schema:- $ref: '#/components/schemas/DeleteTraceResponse'+ $ref: '#/components/schemas/unstableDashboard''400':description: ''content:@@ -6864,54 +8160,44 @@ paths:content:application/json:schema: {}+ '429':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/unstablePublicApiError'+ '500':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/unstablePublicApiError'security: *ref_0requestBody:required: truecontent:application/json:schema:- type: object- properties:- traceIds:- type: array- items:- type: string- description: List of trace IDs to delete- required:- - traceIds- /api/public/unstable/dashboard-widgets:+ $ref: '#/components/schemas/unstableCreateDashboardRequest'+ /api/public/unstable/dashboards/{dashboardId}:get:- description: |-- List dashboard widgets in the project, ordered by most recently- updated first.-- Responses may include legacy `traces` widgets created before this- API existed. New widgets cannot be created with `view: traces`.- operationId: unstable_dashboardWidgets_list+ description: Get a dashboard by id.+ operationId: unstable_dashboards_gettags:- - UnstableDashboardWidgets+ - UnstableDashboardsparameters:- - name: page- in: query- description: 1-based page number. Defaults to `1`.- required: false- schema:- type: integer- nullable: true- - name: limit- in: query- description: Maximum number of items per page. Defaults to `50`.- required: false+ - name: dashboardId+ in: path+ required: trueschema:- type: integer- nullable: true+ type: stringresponses:'200':description: ''content:application/json:schema:- $ref: '#/components/schemas/unstableDashboardWidgetList'+ $ref: '#/components/schemas/unstableDashboard''400':description: ''content:@@ -6950,48 +8236,86 @@ paths:schema:$ref: '#/components/schemas/unstablePublicApiError'security: *ref_0- post:- description: >-- Create a dashboard widget (a standalone chart definition you place on-- any dashboard).--- This endpoint creates the widget only; place it on a dashboard via-- `POST /dashboards/{dashboardId}/placements`.--- Supported views are `observations`, `scores-numeric`, `scores-boolean`,- and `scores-categorical`.-- The legacy `traces` view is not supported by this unstable API.-- Widgets are created as v2 internally.--- `chartConfig` is optional and defaults to the plain config for-- `chartType`; when `chartConfig.type` is given it must match-- `chartType`.--- Unstable API note:-- - This surface may evolve while dashboard/widget APIs are being- finalized.- operationId: unstable_dashboardWidgets_create+ patch:+ description: Update a dashboard's name, description, definition, or filters.+ operationId: unstable_dashboards_update+ tags:+ - UnstableDashboards+ parameters:+ - name: dashboardId+ in: path+ required: true+ schema:+ type: string+ responses:+ '200':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/unstableDashboard'+ '400':+ description: ''+ content:+ application/json:+ schema: {}+ '401':+ description: ''+ content:+ application/json:+ schema: {}+ '403':+ description: ''+ content:+ application/json:+ schema: {}+ '404':+ description: ''+ content:+ application/json:+ schema: {}+ '405':+ description: ''+ content:+ application/json:+ schema: {}+ '429':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/unstablePublicApiError'+ '500':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/unstablePublicApiError'+ security: *ref_0+ requestBody:+ required: true+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/unstableUpdateDashboardRequest'+ delete:+ description: Delete a dashboard.+ operationId: unstable_dashboards_deletetags:- - UnstableDashboardWidgets- parameters: []+ - UnstableDashboards+ parameters:+ - name: dashboardId+ in: path+ required: true+ schema:+ type: stringresponses:'200':description: ''content:application/json:schema:- $ref: '#/components/schemas/unstableDashboardWidget'+ $ref: '#/components/schemas/unstableDeleteDashboardResponse''400':description: ''content:@@ -7030,23 +8354,24 @@ paths:schema:$ref: '#/components/schemas/unstablePublicApiError'security: *ref_0- requestBody:- required: true- content:- application/json:- schema:- $ref: '#/components/schemas/unstableCreateDashboardWidgetRequest'- /api/public/unstable/dashboard-widgets/{widgetId}:- get:+ /api/public/unstable/dashboards/{dashboardId}/placements:+ post:description: |-- Get a dashboard widget by id.+ Add a placement to a dashboard grid (see `DashboardPlacement` for+ grid semantics).- The response may use `view: traces` for legacy widgets.- operationId: unstable_dashboardWidgets_get+ `id` and the position fields are optional: when omitted, the+ placement gets a server-generated id and is appended below all+ existing tiles as a 6x6 tile. Returns the created placement.++ The referenced widget must exist in the same project or be a+ Langfuse-managed widget. The API returns `409` if a placement with+ the same `id` already exists on the dashboard.+ operationId: unstable_dashboards_addPlacementtags:- - UnstableDashboardWidgets+ - UnstableDashboardsparameters:- - name: widgetId+ - name: dashboardIdin: pathrequired: trueschema:@@ -7057,7 +8382,7 @@ paths:content:application/json:schema:- $ref: '#/components/schemas/unstableDashboardWidget'+ $ref: '#/components/schemas/unstableDashboardPlacement''400':description: ''content:@@ -7083,6 +8408,12 @@ paths:content:application/json:schema: {}+ '409':+ description: ''+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/unstablePublicApiError''429':description: ''content:@@ -7096,22 +8427,29 @@ paths:schema:$ref: '#/components/schemas/unstablePublicApiError'security: *ref_0+ requestBody:+ required: true+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/unstableCreateDashboardPlacementRequest'+ /api/public/unstable/dashboards/{dashboardId}/placements/{placementId}:patch:description: |-- Update a dashboard widget.-- All fields are optional; at least one field is required.- Changing `chartType` without sending `chartConfig` resets the config- to the new chart type's defaults. When `chartConfig.type` is given- it must match the widget's (possibly updated) `chartType`.-- `view` cannot be changed to the legacy `traces` value. Existing- `traces` widgets may be updated on other fields.- operationId: unstable_dashboardWidgets_update+ Move or resize a placement. All fields are optional; at least one is+ required. Omitted fields keep their current value. The placement's+ content (widget/preset reference) and id cannot change — delete and+ re-add the placement to swap content. Returns the updated placement.+ operationId: unstable_dashboards_updatePlacementtags:- - UnstableDashboardWidgets+ - UnstableDashboardsparameters:- - name: widgetId+ - name: dashboardId+ in: path+ required: true+ schema:+ type: string+ - name: placementIdin: pathrequired: trueschema:@@ -7122,7 +8460,7 @@ paths:content:application/json:schema:- $ref: '#/components/schemas/unstableDashboardWidget'+ $ref: '#/components/schemas/unstableDashboardPlacement''400':description: ''content:@@ -7166,18 +8504,21 @@ paths:content:application/json:schema:- $ref: '#/components/schemas/unstableUpdateDashboardWidgetRequest'+ $ref: '#/components/schemas/unstableUpdateDashboardPlacementRequest'delete:- description: |-- Delete a dashboard widget.-- The API returns `409` while the widget is still placed on a dashboard.- Remove those placements first.- operationId: unstable_dashboardWidgets_delete+ description: >-+ Remove a placement from a dashboard grid without deleting the referenced+ widget.+ operationId: unstable_dashboards_deletePlacementtags:- - UnstableDashboardWidgets+ - UnstableDashboardsparameters:- - name: widgetId+ - name: dashboardId+ in: path+ required: true+ schema:+ type: string+ - name: placementIdin: pathrequired: trueschema:@@ -7188,7 +8529,7 @@ paths:content:application/json:schema:- $ref: '#/components/schemas/unstableDeleteDashboardWidgetResponse'+ $ref: '#/components/schemas/unstableDeleteDashboardPlacementResponse''400':description: ''content:@@ -7214,12 +8555,6 @@ paths:content:application/json:schema: {}- '409':- description: ''- content:- application/json:- schema:- $ref: '#/components/schemas/unstablePublicApiError''429':description: ''content:@@ -7233,36 +8568,123 @@ paths:schema:$ref: '#/components/schemas/unstablePublicApiError'security: *ref_0- /api/public/unstable/dashboards:- get:- description: |-- List dashboards in the project, ordered by most recently updated- first.- operationId: unstable_dashboards_list- tags:- - UnstableDashboards- parameters:- - name: page- in: query- description: 1-based page number. Defaults to `1`.- required: false- schema:- type: integer- nullable: true- - name: limit- in: query- description: Maximum number of items per page. Defaults to `50`.- required: false- schema:- type: integer- nullable: true+ /api/public/unstable/evaluation-rules:+ post:+ description: >-+ **Deprecated:** On Langfuse Cloud, this unstable endpoint is deprecated+ and will be removed on November 16, 2026. Use the stable+ `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments+ are unaffected by this date; the endpoint becomes unavailable when they+ upgrade to Langfuse v4. See the [Langfuse v3 to v4 upgrade+ guide](https://langfuse.com/self-hosting/upgrade/upgrade-guides/upgrade-v3-to-v4).+++ Create an evaluation rule.+++ An evaluation rule defines **what** incoming data should be evaluated+ and **how prompt variables should be populated** from that data.+++ Use this resource after choosing an evaluator from the evaluator+ endpoints.+++ Key rules:++ - `name` must be unique within the project for public evaluation rules++ - `target` must be `observation` or `experiment`++ - provide either the compatibility `evaluator` field or the new+ `evaluators` array, never both++ - each evaluator `name` + `type` must identify an existing evaluator+ family returned by the evaluator endpoints++ - Langfuse resolves that family to its latest version before saving the+ evaluation rule++ - for `target=experiment`, use dataset `id` values from `GET+ /api/public/v2/datasets` when filtering by `datasetId`++ - an omitted LLM-as-judge assignment mapping inherits the evaluator+ version's default mapping++ - the effective mapping must map every evaluator prompt variable exactly+ once++ - for `code` evaluators, Langfuse uses the fixed code runtime mapping;+ omit `mapping` in create and update requests++ - for user-provided `llm_as_judge` mappings, `expected_output` and+ `experiment_item_metadata` are only valid for `target=experiment`++ - if `enabled=true`, Langfuse validates that the referenced evaluator+ can currently run++ - at most 500 evaluation rules can be effectively active in one project+ at the same time (enforced identically by the API, the MCP tools, and+ the app)+++ If an evaluation rule with the same `name` already exists in the+ project, the API returns `409`.++ In that case, update the existing resource with `PATCH+ /api/public/unstable/evaluation-rules/{evaluationRuleId}` instead of+ creating a second one.+++ If enabling this resource would exceed the 500-active limit, the API+ also returns `409`.++ In that case, disable or pause another active evaluation rule before+ enabling a new one.+++ Current scope:++ - evaluation rules are live-ingestion rules only++ - they do not trigger historical backfills+++ Recovery guidance:++ - `400 invalid_filter_value`: fix the filter `column` or `value` using+ `details.column`, `details.invalidValues`, and `details.allowedValues`++ - `400 invalid_filter_value` with `details.column=datasetId`: call `GET+ /api/public/v2/datasets`, then retry with dataset `id` values from that+ response++ - `400 missing_variable_mapping`: for `llm_as_judge` evaluators, fetch+ the evaluator again and make sure every variable in `variables` appears+ exactly once in `mapping`++ - `400 duplicate_variable_mapping`: remove repeated mappings for the+ same variable++ - `400 invalid_variable_mapping`: for `llm_as_judge`, switch to a valid+ `source` for the selected `target`, or fix the variable name++ - `400 invalid_json_path`: remove or correct the `jsonPath`++ - `422 evaluator_preflight_failed`: the selected evaluator cannot run+ with the resolved model configuration. Fix the evaluator/default model+ setup, then retry the create request.+ operationId: unstable_evaluationRules_create+ tags:+ - UnstableEvaluationRules+ parameters: []responses:'200':description: ''content:application/json:schema:- $ref: '#/components/schemas/unstableDashboardList'+ $ref: '#/components/schemas/unstableEvaluationRule''400':description: ''content:@@ -7288,57 +8710,18 @@ paths:content:application/json:schema: {}- '429':+ '409':description: ''content:application/json:schema:$ref: '#/components/schemas/unstablePublicApiError'- '500':+ '422':description: ''content:application/json:schema:$ref: '#/components/schemas/unstablePublicApiError'- security: *ref_0- post:- description: Create a dashboard.- operationId: unstable_dashboards_create- tags:- - UnstableDashboards- parameters: []- responses:- '200':- description: ''- content:- application/json:- schema:- $ref: '#/components/schemas/unstableDashboard'- '400':- description: ''- content:- application/json:- schema: {}- '401':- description: ''- content:- application/json:- schema: {}- '403':- description: ''- content:- application/json:- schema: {}- '404':- description: ''- content:- application/json:- schema: {}- '405':- description: ''- content:- application/json:- schema: {}'429':description: ''content:@@ -7357,82 +8740,50 @@ paths:content:application/json:schema:- $ref: '#/components/schemas/unstableCreateDashboardRequest'- /api/public/unstable/dashboards/{dashboardId}:+ $ref: '#/components/schemas/unstableCreateEvaluationRuleRequest'+ deprecated: trueget:- description: Get a dashboard by id.- operationId: unstable_dashboards_get+ description: >-+ **Deprecated:** On Langfuse Cloud, this unstable endpoint is deprecated+ and will be removed on November 16, 2026. Use the stable+ `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments+ are unaffected by this date; the endpoint becomes unavailable when they+ upgrade to Langfuse v4. See the [Langfuse v3 to v4 upgrade+ guide](https://langfuse.com/self-hosting/upgrade/upgrade-guides/upgrade-v3-to-v4).+++ List evaluation rules in the authenticated project.+++ This includes legacy `trace` and `dataset` rules so they can be+ inspected and migrated to v4 rules. Legacy rules are read-only through+ this API; create, update, and delete continue to support only+ `observation` and `experiment` rules.+ operationId: unstable_evaluationRules_listtags:- - UnstableDashboards+ - UnstableEvaluationRulesparameters:- - name: dashboardId- in: path- required: true+ - name: page+ in: query+ description: 1-based page number. Defaults to `1`.+ required: falseschema:- type: string- responses:- '200':- description: ''- content:- application/json:- schema:- $ref: '#/components/schemas/unstableDashboard'- '400':- description: ''- content:- application/json:- schema: {}- '401':- description: ''- content:- application/json:- schema: {}- '403':- description: ''- content:- application/json:- schema: {}- '404':- description: ''- content:- application/json:- schema: {}- '405':- description: ''- content:- application/json:- schema: {}- '429':- description: ''- content:- application/json:- schema:- $ref: '#/components/schemas/unstablePublicApiError'- '500':- description: ''- content:- application/json:- schema:- $ref: '#/components/schemas/unstablePublicApiError'- security: *ref_0- patch:- description: Update a dashboard's name, description, definition, or filters.- operationId: unstable_dashboards_update- tags:- - UnstableDashboards- parameters:- - name: dashboardId- in: path- required: true+ type: integer+ nullable: true+ - name: limit+ in: query+ description: Maximum number of items per page. Defaults to `50`.+ required: falseschema:- type: string+ type: integer+ nullable: trueresponses:'200':description: ''content:application/json:schema:- $ref: '#/components/schemas/unstableDashboard'+ $ref: '#/components/schemas/unstableEvaluationRules''400':description: ''content:@@ -7471,20 +8822,33 @@ paths:schema:$ref: '#/components/schemas/unstablePublicApiError'security: *ref_0- requestBody:- required: true- content:- application/json:- schema:- $ref: '#/components/schemas/unstableUpdateDashboardRequest'- delete:- description: Delete a dashboard.- operationId: unstable_dashboards_delete+ deprecated: true+ /api/public/unstable/evaluation-rules/{evaluationRuleId}:+ get:+ description: >-+ **Deprecated:** On Langfuse Cloud, this unstable endpoint is deprecated+ and will be removed on November 16, 2026. Use the stable+ `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments+ are unaffected by this date; the endpoint becomes unavailable when they+ upgrade to Langfuse v4. See the [Langfuse v3 to v4 upgrade+ guide](https://langfuse.com/self-hosting/upgrade/upgrade-guides/upgrade-v3-to-v4).+++ Get one evaluation rule by its identifier.+++ Use this endpoint to inspect the current evaluator, target, mapping,+ filters, execution timing, and effective runtime status. Legacy `trace`+ and `dataset` rules are returned for migration and are read-only through+ this API.+ operationId: unstable_evaluationRules_gettags:- - UnstableDashboards+ - UnstableEvaluationRulesparameters:- - name: dashboardId+ - name: evaluationRuleIdin: path+ description: >-+ Evaluation rule identifier returned by the evaluation rule endpoints.required: trueschema:type: string@@ -7494,7 +8858,7 @@ paths:content:application/json:schema:- $ref: '#/components/schemas/unstableDeleteDashboardResponse'+ $ref: '#/components/schemas/unstableReadableEvaluationRule''400':description: ''content:@@ -7533,25 +8897,73 @@ paths:schema:$ref: '#/components/schemas/unstablePublicApiError'security: *ref_0- /api/public/unstable/dashboards/{dashboardId}/placements:- post:- description: |-- Add a placement to a dashboard grid (see `DashboardPlacement` for- grid semantics).+ deprecated: true+ patch:+ description: >-+ **Deprecated:** On Langfuse Cloud, this unstable endpoint is deprecated+ and will be removed on November 16, 2026. Use the stable+ `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments+ are unaffected by this date; the endpoint becomes unavailable when they+ upgrade to Langfuse v4. See the [Langfuse v3 to v4 upgrade+ guide](https://langfuse.com/self-hosting/upgrade/upgrade-guides/upgrade-v3-to-v4).- `id` and the position fields are optional: when omitted, the- placement gets a server-generated id and is appended below all- existing tiles as a 6x6 tile. Returns the created placement.- The referenced widget must exist in the same project or be a- Langfuse-managed widget. The API returns `409` if a placement with- the same `id` already exists on the dashboard.- operationId: unstable_dashboards_addPlacement+ Update an evaluation rule.+++ Typical uses:++ - enable or disable live execution++ - switch to another evaluator++ - adjust sampling++ - change filters++ - update LLM-as-judge variable mappings+++ Important behavior:++ - provide only the fields you want to change++ - if you provide `evaluator`, Langfuse resolves that evaluator family to+ its latest version before saving++ - changing `target`, `filter`, or an LLM-as-judge `mapping` must still+ produce a valid target-specific configuration++ - if you change `target` for an LLM-as-judge rule, also send a+ compatible `filter` and `mapping` in the same request unless the+ existing ones are still valid for the new target++ - for `code` evaluator rules, omit `mapping`; Langfuse stores the fixed+ code runtime mapping automatically++ - if the resulting config is enabled, Langfuse re-validates that the+ selected evaluator can run++ - if the update would move a non-active evaluation rule into the active+ state and the project already has 500 active evaluation rules, the API+ returns `409`+++ Recovery guidance:++ - if an LLM-as-judge update fails with `missing_variable_mapping` or+ `invalid_variable_mapping` after changing `evaluator` or `target`,+ resend the request with a complete new `mapping`++ - if the update fails with `invalid_filter_value` after changing+ `target`, resend the request with a target-compatible `filter`+ operationId: unstable_evaluationRules_updatetags:- - UnstableDashboards+ - UnstableEvaluationRulesparameters:- - name: dashboardId+ - name: evaluationRuleIdin: path+ description: Evaluation rule identifier.required: trueschema:type: string@@ -7561,7 +8973,7 @@ paths:content:application/json:schema:- $ref: '#/components/schemas/unstableDashboardPlacement'+ $ref: '#/components/schemas/unstableEvaluationRule''400':description: ''content:@@ -7587,84 +8999,12 @@ paths:content:application/json:schema: {}- '409':- description: ''- content:- application/json:- schema:- $ref: '#/components/schemas/unstablePublicApiError'- '429':- description: ''- content:- application/json:- schema:- $ref: '#/components/schemas/unstablePublicApiError'- '500':+ '422':description: ''content:application/json:schema:$ref: '#/components/schemas/unstablePublicApiError'- security: *ref_0- requestBody:- required: true- content:- application/json:- schema:- $ref: '#/components/schemas/unstableCreateDashboardPlacementRequest'- /api/public/unstable/dashboards/{dashboardId}/placements/{placementId}:- patch:- description: |-- Move or resize a placement. All fields are optional; at least one is- required. Omitted fields keep their current value. The placement's- content (widget/preset reference) and id cannot change — delete and- re-add the placement to swap content. Returns the updated placement.- operationId: unstable_dashboards_updatePlacement- tags:- - UnstableDashboards- parameters:- - name: dashboardId- in: path- required: true- schema:- type: string- - name: placementId- in: path- required: true- schema:- type: string- responses:- '200':- description: ''- content:- application/json:- schema:- $ref: '#/components/schemas/unstableDashboardPlacement'- '400':- description: ''- content:- application/json:- schema: {}- '401':- description: ''- content:- application/json:- schema: {}- '403':- description: ''- content:- application/json:- schema: {}- '404':- description: ''- content:- application/json:- schema: {}- '405':- description: ''- content:- application/json:- schema: {}'429':description: ''content:@@ -7683,22 +9023,30 @@ paths:content:application/json:schema:- $ref: '#/components/schemas/unstableUpdateDashboardPlacementRequest'+ $ref: '#/components/schemas/unstableUpdateEvaluationRuleRequest'+ deprecated: truedelete:description: >-- Remove a placement from a dashboard grid without deleting the referenced- widget.- operationId: unstable_dashboards_deletePlacement+ **Deprecated:** On Langfuse Cloud, this unstable endpoint is deprecated+ and will be removed on November 16, 2026. Use the stable+ `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments+ are unaffected by this date; the endpoint becomes unavailable when they+ upgrade to Langfuse v4. See the [Langfuse v3 to v4 upgrade+ guide](https://langfuse.com/self-hosting/upgrade/upgrade-guides/upgrade-v3-to-v4).+++ Delete an evaluation rule.+++ This removes the live-ingestion rule only. It does not delete the+ referenced evaluator.+ operationId: unstable_evaluationRules_deletetags:- - UnstableDashboards+ - UnstableEvaluationRulesparameters:- - name: dashboardId- in: path- required: true- schema:- type: string- - name: placementId+ - name: evaluationRuleIdin: path+ description: Evaluation rule identifier.required: trueschema:type: string@@ -7708,7 +9056,7 @@ paths:content:application/json:schema:- $ref: '#/components/schemas/unstableDeleteDashboardPlacementResponse'+ $ref: '#/components/schemas/unstableDeleteEvaluationRuleResponse''400':description: ''content:@@ -7747,107 +9095,94 @@ paths:schema:$ref: '#/components/schemas/unstablePublicApiError'security: *ref_0- /api/public/unstable/evaluation-rules:+ deprecated: true+ /api/public/unstable/evaluators:post:description: >-- Create an evaluation rule.--- An evaluation rule defines **what** incoming data should be evaluated- and **how prompt variables should be populated** from that data.--- Use this resource after choosing an evaluator from the evaluator- endpoints.--- Key rules:+ **Deprecated:** On Langfuse Cloud, this unstable endpoint is deprecated+ and will be removed on November 16, 2026. Use the stable+ `/api/public/v2/evaluators` API instead. Self-hosted deployments are+ unaffected by this date; the endpoint becomes unavailable when they+ upgrade to Langfuse v4. See the [Langfuse v3 to v4 upgrade+ guide](https://langfuse.com/self-hosting/upgrade/upgrade-guides/upgrade-v3-to-v4).- - `name` must be unique within the project for public evaluation rules- - `target` must be `observation` or `experiment`+ Create an evaluator in the authenticated project.- - provide either the compatibility `evaluator` field or the new- `evaluators` array, never both- - each evaluator `name` + `type` must identify an existing evaluator- family returned by the evaluator endpoints+ Use evaluators to define **how** Langfuse should score data.- - Langfuse resolves that family to its latest version before saving the- evaluation rule+ LLM-as-a-judge evaluators define a prompt, expected structured output,+ and optional model configuration.- - for `target=experiment`, use dataset `id` values from `GET- /api/public/v2/datasets` when filtering by `datasetId`+ Code evaluators define source code and a runtime language.- - an omitted LLM-as-judge assignment mapping inherits the evaluator- version's default mapping- - the effective mapping must map every evaluator prompt variable exactly- once+ Naming behavior:- - for `code` evaluators, Langfuse uses the fixed code runtime mapping;- omit `mapping` in create and update requests+ - If this is a new evaluator name in your project, Langfuse creates+ version `1`.- - for user-provided `llm_as_judge` mappings, `expected_output` and- `experiment_item_metadata` are only valid for `target=experiment`+ - If the name already exists in your project, Langfuse creates the next+ version and returns it.- - if `enabled=true`, Langfuse validates that the referenced evaluator- can currently run+ - The evaluator `id` remains stable across versions.- - at most 500 evaluation rules can be effectively active in one project- at the same time (enforced identically by the API, the MCP tools, and- the app)+ - Existing evaluation rules automatically use the latest evaluator+ version; no rule update is required.- If an evaluation rule with the same `name` already exists in the- project, the API returns `409`.+ Recommended workflow:- In that case, update the existing resource with `PATCH- /api/public/unstable/evaluation-rules/{evaluationRuleId}` instead of- creating a second one.+ 1. Create the evaluator.+ 2. Read the returned `variables` array.- If enabling this resource would exceed the 500-active limit, the API- also returns `409`.+ 3. Read the returned `outputDefinition.dataType` so the client knows+ whether future scores will be numeric, boolean, or categorical.- In that case, disable or pause another active evaluation rule before- enabling a new one.+ 4. Create one or more evaluation rules that reference the returned+ evaluator family using `name` and `type`.- Current scope:+ Code evaluator validation:- - evaluation rules are live-ingestion rules only+ - At creation, Langfuse only validates the request shape- - they do not trigger historical backfills+ - The `sourceCode` itself is not executed here. It is first run+ (preflight-tested against a sample observation) when you link the+ evaluator to an evaluation rule, so runtime errors in the code surface+ at evaluation-rule creation, not at evaluator creation.Recovery guidance:- - `400 invalid_filter_value`: fix the filter `column` or `value` using- `details.column`, `details.invalidValues`, and `details.allowedValues`+ - `422` with `code=evaluator_preflight_failed`: the evaluator cannot run+ with the resolved model configuration. Add a valid explicit+ `modelConfig`, or configure the project's default evaluation model, then+ retry the same request.- - `400 invalid_filter_value` with `details.column=datasetId`: call `GET- /api/public/v2/datasets`, then retry with dataset `id` values from that- response+ - `400` with `code=invalid_body`: the request shape is malformed. Use+ the structured `details.issues` array to fix the specific fields and+ retry.- - `400 missing_variable_mapping`: for `llm_as_judge` evaluators, fetch- the evaluator again and make sure every variable in `variables` appears- exactly once in `mapping`+ - `400` with `code=invalid_body` on `outputDefinition`: for+ `type=llm_as_judge`, send `dataType`, `reasoning.description`, and+ `score.description`. Do not send `version`; it is not part of the public+ request shape.- - `400 duplicate_variable_mapping`: remove repeated mappings for the- same variable+ - If `type` is omitted, Langfuse treats the request as+ `type=llm_as_judge` for backwards compatibility. New clients should send+ `type` explicitly.- - `400 invalid_variable_mapping`: for `llm_as_judge`, switch to a valid- `source` for the selected `target`, or fix the variable name- - `400 invalid_json_path`: remove or correct the `jsonPath`+ Unstable API note:- - `422 evaluator_preflight_failed`: the selected evaluator cannot run- with the resolved model configuration. Fix the evaluator/default model- setup, then retry the create request.- operationId: unstable_evaluationRules_create+ - This surface may evolve while the underlying evaluation data model is+ being redesigned.+ operationId: unstable_evaluators_createtags:- - UnstableEvaluationRules+ - UnstableEvaluatorsparameters: []responses:'200':@@ -7855,7 +9190,7 @@ paths:content:application/json:schema:- $ref: '#/components/schemas/unstableEvaluationRule'+ $ref: '#/components/schemas/unstableEvaluator''400':description: ''content:@@ -7911,19 +9246,20 @@ paths:content:application/json:schema:- $ref: '#/components/schemas/unstableCreateEvaluationRuleRequest'+ $ref: '#/components/schemas/unstableCreateEvaluatorRequest'+ deprecated: trueget:- description: >-- List evaluation rules in the authenticated project.+ description: |-+ **Deprecated:** On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluators` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4. See the [Langfuse v3 to v4 upgrade guide](https://langfuse.com/self-hosting/upgrade/upgrade-guides/upgrade-v3-to-v4).+ List the evaluators available to the authenticated project.- This includes legacy `trace` and `dataset` rules so they can be- inspected and migrated to v4 rules. Legacy rules are read-only through- this API; create, update, and delete continue to support only- `observation` and `experiment` rules.- operationId: unstable_evaluationRules_list+ Important behavior:+ - This endpoint returns the latest version of each available evaluator.+ - Every evaluator is owned by the authenticated project.+ operationId: unstable_evaluators_listtags:- - UnstableEvaluationRules+ - UnstableEvaluatorsparameters:- name: pagein: query@@ -7945,7 +9281,7 @@ paths:content:application/json:schema:- $ref: '#/components/schemas/unstableEvaluationRules'+ $ref: '#/components/schemas/unstableEvaluators''400':description: ''content:@@ -7984,24 +9320,32 @@ paths:schema:$ref: '#/components/schemas/unstablePublicApiError'security: *ref_0- /api/public/unstable/evaluation-rules/{evaluationRuleId}:+ deprecated: true+ /api/public/unstable/evaluators/{evaluatorId}:get:description: >-- Get one evaluation rule by its identifier.-+ **Deprecated:** On Langfuse Cloud, this unstable endpoint is deprecated+ and will be removed on November 16, 2026. Use the stable+ `/api/public/v2/evaluators` API instead. Self-hosted deployments are+ unaffected by this date; the endpoint becomes unavailable when they+ upgrade to Langfuse v4. See the [Langfuse v3 to v4 upgrade+ guide](https://langfuse.com/self-hosting/upgrade/upgrade-guides/upgrade-v3-to-v4).- Use this endpoint to inspect the current evaluator, target, mapping,- filters, execution timing, and effective runtime status. Legacy `trace`- and `dataset` rules are returned for migration and are read-only through- this API.- operationId: unstable_evaluationRules_get++ Get one evaluator by `id`.+++ This endpoint always returns the evaluator's latest version. Use it when+ you want the current prompt, output definition, model configuration, and+ derived variables for the evaluator you plan to use in an evaluation+ rule.+ operationId: unstable_evaluators_gettags:- - UnstableEvaluationRules+ - UnstableEvaluatorsparameters:- - name: evaluationRuleId+ - name: evaluatorIdin: path- description: >-- Evaluation rule identifier returned by the evaluation rule endpoints.+ description: Evaluator identifier returned by the evaluator endpoints.required: trueschema:type: string@@ -8011,7 +9355,7 @@ paths:content:application/json:schema:- $ref: '#/components/schemas/unstableReadableEvaluationRule'+ $ref: '#/components/schemas/unstableEvaluator''400':description: ''content:@@ -8050,64 +9394,35 @@ paths:schema:$ref: '#/components/schemas/unstablePublicApiError'security: *ref_0- patch:+ deprecated: true+ delete:description: >-- Update an evaluation rule.--- Typical uses:-- - enable or disable live execution-- - switch to another evaluator-- - adjust sampling+ **Deprecated:** On Langfuse Cloud, this unstable endpoint is deprecated+ and will be removed on November 16, 2026. Use the stable+ `/api/public/v2/evaluators` API instead. Self-hosted deployments are+ unaffected by this date; the endpoint becomes unavailable when they+ upgrade to Langfuse v4. See the [Langfuse v3 to v4 upgrade+ guide](https://langfuse.com/self-hosting/upgrade/upgrade-guides/upgrade-v3-to-v4).- - change filters- - update LLM-as-judge variable mappings+ Delete an evaluator.Important behavior:- - provide only the fields you want to change-- - if you provide `evaluator`, Langfuse resolves that evaluator family to- its latest version before saving-- - changing `target`, `filter`, or an LLM-as-judge `mapping` must still- produce a valid target-specific configuration-- - if you change `target` for an LLM-as-judge rule, also send a- compatible `filter` and `mapping` in the same request unless the- existing ones are still valid for the new target-- - for `code` evaluator rules, omit `mapping`; Langfuse stores the fixed- code runtime mapping automatically-- - if the resulting config is enabled, Langfuse re-validates that the- selected evaluator can run-- - if the update would move a non-active evaluation rule into the active- state and the project already has 500 active evaluation rules, the API- returns `409`--- Recovery guidance:+ - This deletes the evaluator including all of its stored versions.- - if an LLM-as-judge update fails with `missing_variable_mapping` or- `invalid_variable_mapping` after changing `evaluator` or `target`,- resend the request with a complete new `mapping`+ - Evaluation rule assignments referencing the evaluator are also+ deleted.- - if the update fails with `invalid_filter_value` after changing- `target`, resend the request with a target-compatible `filter`- operationId: unstable_evaluationRules_update+ - Scores already produced by the evaluator are not deleted.+ operationId: unstable_evaluators_deletetags:- - UnstableEvaluationRules+ - UnstableEvaluatorsparameters:- - name: evaluationRuleId+ - name: evaluatorIdin: path- description: Evaluation rule identifier.+ description: Evaluator identifier returned by the evaluator endpoints.required: trueschema:type: string@@ -8117,7 +9432,7 @@ paths:content:application/json:schema:- $ref: '#/components/schemas/unstableEvaluationRule'+ $ref: '#/components/schemas/unstableDeleteEvaluatorResponse''400':description: ''content:@@ -8143,12 +9458,6 @@ paths:content:application/json:schema: {}- '422':- description: ''- content:- application/json:- schema:- $ref: '#/components/schemas/unstablePublicApiError''429':description: ''content:@@ -8162,1919 +9471,2974 @@ paths:schema:$ref: '#/components/schemas/unstablePublicApiError'security: *ref_0- requestBody:- required: true- content:- application/json:- schema:- $ref: '#/components/schemas/unstableUpdateEvaluationRuleRequest'- delete:+ deprecated: true+components:+ schemas:+ AnnotationQueueStatus:+ title: AnnotationQueueStatus+ type: string+ enum:+ - PENDING+ - COMPLETED+ AnnotationQueueObjectType:+ title: AnnotationQueueObjectType+ type: string+ enum:+ - TRACE+ - OBSERVATION+ - SESSION+ AnnotationQueue:+ title: AnnotationQueue+ type: object+ properties:+ id:+ type: string+ name:+ type: string+ description:+ type: string+ nullable: true+ scoreConfigIds:+ type: array+ items:+ type: string+ createdAt:+ type: string+ format: date-time+ updatedAt:+ type: string+ format: date-time+ required:+ - id+ - name+ - description+ - scoreConfigIds+ - createdAt+ - updatedAt+ AnnotationQueueItem:+ title: AnnotationQueueItem+ type: object+ properties:+ id:+ type: string+ queueId:+ type: string+ objectId:+ type: string+ objectType:+ $ref: '#/components/schemas/AnnotationQueueObjectType'+ status:+ $ref: '#/components/schemas/AnnotationQueueStatus'+ completedAt:+ type: string+ format: date-time+ nullable: true+ createdAt:+ type: string+ format: date-time+ updatedAt:+ type: string+ format: date-time+ required:+ - id+ - queueId+ - objectId+ - objectType+ - status+ - createdAt+ - updatedAt+ PaginatedAnnotationQueues:+ title: PaginatedAnnotationQueues+ type: object+ properties:+ data:+ type: array+ items:+ $ref: '#/components/schemas/AnnotationQueue'+ meta:+ $ref: '#/components/schemas/utilsMetaResponse'+ required:+ - data+ - meta+ PaginatedAnnotationQueueItems:+ title: PaginatedAnnotationQueueItems+ type: object+ properties:+ data:+ type: array+ items:+ $ref: '#/components/schemas/AnnotationQueueItem'+ meta:+ $ref: '#/components/schemas/utilsMetaResponse'+ required:+ - data+ - meta+ CreateAnnotationQueueRequest:+ title: CreateAnnotationQueueRequest+ type: object+ properties:+ name:+ type: string+ description:+ type: string+ nullable: true+ scoreConfigIds:+ type: array+ items:+ type: string+ required:+ - name+ - scoreConfigIds+ CreateAnnotationQueueItemRequest:+ title: CreateAnnotationQueueItemRequest+ type: object+ properties:+ objectId:+ type: string+ objectType:+ $ref: '#/components/schemas/AnnotationQueueObjectType'+ status:+ $ref: '#/components/schemas/AnnotationQueueStatus'+ nullable: true+ description: Defaults to PENDING for new queue items+ required:+ - objectId+ - objectType+ UpdateAnnotationQueueItemRequest:+ title: UpdateAnnotationQueueItemRequest+ type: object+ properties:+ status:+ $ref: '#/components/schemas/AnnotationQueueStatus'+ nullable: true+ DeleteAnnotationQueueItemResponse:+ title: DeleteAnnotationQueueItemResponse+ type: object+ properties:+ success:+ type: boolean+ message:+ type: string+ required:+ - success+ - message+ AnnotationQueueAssignmentRequest:+ title: AnnotationQueueAssignmentRequest+ type: object+ properties:+ userId:+ type: string+ required:+ - userId+ DeleteAnnotationQueueAssignmentResponse:+ title: DeleteAnnotationQueueAssignmentResponse+ type: object+ properties:+ success:+ type: boolean+ required:+ - success+ CreateAnnotationQueueAssignmentResponse:+ title: CreateAnnotationQueueAssignmentResponse+ type: object+ properties:+ userId:+ type: string+ queueId:+ type: string+ projectId:+ type: string+ required:+ - userId+ - queueId+ - projectId+ BlobStorageIntegrationType:+ title: BlobStorageIntegrationType+ type: string+ enum:+ - S3+ - S3_COMPATIBLE+ - AZURE_BLOB_STORAGE+ BlobStorageIntegrationFileType:+ title: BlobStorageIntegrationFileType+ type: string+ enum:+ - JSON+ - CSV+ - JSONL+ - PARQUETdescription: >-- Delete an evaluation rule.--- This removes the live-ingestion rule only. It does not delete the- referenced evaluator.- operationId: unstable_evaluationRules_delete- tags:- - UnstableEvaluationRules- parameters:- - name: evaluationRuleId- in: path- description: Evaluation rule identifier.- required: true- schema:- type: string- responses:- '200':- description: ''- content:- application/json:- schema:- $ref: '#/components/schemas/unstableDeleteEvaluationRuleResponse'- '400':- description: ''- content:- application/json:- schema: {}- '401':- description: ''- content:- application/json:- schema: {}- '403':- description: ''- content:- application/json:- schema: {}- '404':- description: ''- content:- application/json:- schema: {}- '405':- description: ''- content:- application/json:- schema: {}- '429':- description: ''- content:- application/json:- schema:- $ref: '#/components/schemas/unstablePublicApiError'- '500':- description: ''- content:- application/json:- schema:- $ref: '#/components/schemas/unstablePublicApiError'- security: *ref_0- /api/public/unstable/evaluators:- post:+ File format for exported data. `PARQUET` is a columnar binary format+ encoded and compressed by the storage engine; gzip compression does not+ apply to it. Note that the model-price columns (`input_price`,+ `output_price`, `total_price`) are not included in Parquet observation+ exports.+ BlobStorageIntegrationFileTypeResponse:+ title: BlobStorageIntegrationFileTypeResponse+ type: string+ enum:+ - JSON+ - CSV+ - JSONL+ - PARQUET+ description: File type reported for an existing integration.+ BlobStorageExportMode:+ title: BlobStorageExportMode+ type: string+ enum:+ - FULL_HISTORY+ - FROM_TODAY+ - FROM_CUSTOM_DATE+ BlobStorageExportFrequency:+ title: BlobStorageExportFrequency+ type: string+ enum:+ - every_20_minutes+ - hourly+ - daily+ - weekly+ BlobStorageExportSource:+ title: BlobStorageExportSource+ type: string+ enum:+ - LEGACY_TRACES_OBSERVATIONS+ - OBSERVATIONS_V2+ - LEGACY_TRACES_AND_ENRICHED_OBSERVATIONSdescription: >-- Create an evaluator in the authenticated project.--- Use evaluators to define **how** Langfuse should score data.-- LLM-as-a-judge evaluators define a prompt, expected structured output,- and optional model configuration.-- Code evaluators define source code and a runtime language.--- Naming behavior:-- - If this is a new evaluator name in your project, Langfuse creates- version `1`.-- - If the name already exists in your project, Langfuse creates the next- version and returns it.-- - The evaluator `id` remains stable across versions.-- - Existing evaluation rules automatically use the latest evaluator- version; no rule update is required.--- Recommended workflow:-- 1. Create the evaluator.-- 2. Read the returned `variables` array.-- 3. Read the returned `outputDefinition.dataType` so the client knows- whether future scores will be numeric, boolean, or categorical.-- 4. Create one or more evaluation rules that reference the returned- evaluator family using `name` and `type`.--- Code evaluator validation:+ What data the integration exports.- - At creation, Langfuse only validates the request shape+ - `LEGACY_TRACES_OBSERVATIONS`: traces, observations, and scores tables.+ Observation columns are controlled by `exportFieldGroups`; field groups+ without a counterpart in this data model (e.g. `trace_context`) are+ omitted.- - The `sourceCode` itself is not executed here. It is first run- (preflight-tested against a sample observation) when you link the- evaluator to an evaluation rule, so runtime errors in the code surface- at evaluation-rule creation, not at evaluator creation.+ - `OBSERVATIONS_V2`: same data model as the+ `/api/public/v2/observations` endpoint, plus scores. Columns are+ controlled by `exportFieldGroups`.+ - `LEGACY_TRACES_AND_ENRICHED_OBSERVATIONS`: both sets. Observation+ columns of both portions are controlled by `exportFieldGroups`.- Recovery guidance:- - `422` with `code=evaluator_preflight_failed`: the evaluator cannot run- with the resolved model configuration. Add a valid explicit- `modelConfig`, or configure the project's default evaluation model, then- retry the same request.+ **Note:** which sources a deployment accepts depends on how far it has+ moved to the v4 data model. `OBSERVATIONS_V2` and the+ enriched-observations portion of+ `LEGACY_TRACES_AND_ENRICHED_OBSERVATIONS` read the enriched observations+ table, so they require a deployment that already populates it.+ `LEGACY_TRACES_OBSERVATIONS` and the legacy portion of+ `LEGACY_TRACES_AND_ENRICHED_OBSERVATIONS` read the legacy traces and+ observations tables, so they require a deployment that still populates+ those. A deployment part-way through the migration populates both and+ accepts every source. Selecting a source the deployment cannot serve is+ rejected with `400`, rather than exporting an empty result. See+ https://langfuse.com/docs/v4.+ BlobStorageExportFieldGroup:+ title: BlobStorageExportFieldGroup+ type: string+ enum:+ - core+ - basic+ - time+ - io+ - metadata+ - model+ - usage+ - prompt+ - metrics+ - tools+ - trace_context+ description: >-+ Field group selecting which observation columns are included in the+ export. Applies to all export sources; groups without a counterpart in+ the legacy data model (e.g. `trace_context`) are omitted from the legacy+ observations export.+ CreateBlobStorageIntegrationRequest:+ title: CreateBlobStorageIntegrationRequest+ type: object+ properties:+ projectId:+ type: string+ description: ID of the project in which to configure the blob storage integration+ type:+ $ref: '#/components/schemas/BlobStorageIntegrationType'+ bucketName:+ type: string+ description: >-+ Name of the storage bucket. For AZURE_BLOB_STORAGE, must be a valid+ Azure container name (3-63 chars, lowercase letters, numbers, and+ hyphens only, must start and end with a letter or number, no+ consecutive hyphens).+ endpoint:+ type: string+ nullable: true+ description: Custom endpoint URL (required for S3_COMPATIBLE type)+ region:+ type: string+ description: Storage region+ accessKeyId:+ type: string+ nullable: true+ description: Access key ID for authentication+ secretAccessKey:+ type: string+ nullable: true+ description: Secret access key for authentication (will be encrypted when stored)+ prefix:+ type: string+ nullable: true+ description: >-+ Path prefix for exported files (must end with forward slash if+ provided)+ exportFrequency:+ $ref: '#/components/schemas/BlobStorageExportFrequency'+ enabled:+ type: boolean+ description: Whether the integration is active+ forcePathStyle:+ type: boolean+ description: Use path-style URLs for S3 requests+ fileType:+ $ref: '#/components/schemas/BlobStorageIntegrationFileType'+ exportMode:+ $ref: '#/components/schemas/BlobStorageExportMode'+ exportStartDate:+ type: string+ format: date-time+ nullable: true+ description: >-+ Custom start date for exports (required when exportMode is+ FROM_CUSTOM_DATE). Must not be in the future (27 h tolerance for+ timezone differences).+ compressed:+ type: boolean+ nullable: true+ description: >-+ Enable gzip compression for exported files (.csv.gz, .json.gz,+ .jsonl.gz). Defaults to true.+ exportSource:+ $ref: '#/components/schemas/BlobStorageExportSource'+ nullable: true+ description: >-+ Data to export. When omitted on update, the existing value is+ preserved. When omitted on create, the default is `OBSERVATIONS_V2`+ on Langfuse Cloud, and on self-hosted deployments+ `LEGACY_TRACES_OBSERVATIONS` — or `OBSERVATIONS_V2` where the+ deployment no longer populates the legacy tables. The default is+ never a source the deployment cannot serve. Required when+ `exportFieldGroups` is provided.- - `400` with `code=invalid_body`: the request shape is malformed. Use- the structured `details.issues` array to fix the specific fields and- retry.- - `400` with `code=invalid_body` on `outputDefinition`: for- `type=llm_as_judge`, send `dataType`, `reasoning.description`, and- `score.description`. Do not send `version`; it is not part of the public- request shape.+ **Cloud-only project deprecation gate (effective 2026-05-20):** For+ projects created on or after 2026-05-20 on Langfuse Cloud,+ `LEGACY_TRACES_OBSERVATIONS` and+ `LEGACY_TRACES_AND_ENRICHED_OBSERVATIONS` are rejected with HTTP+ 400. Use `OBSERVATIONS_V2` for all new integrations. Self-hosted+ deployments are unaffected.- - If `type` is omitted, Langfuse treats the request as- `type=llm_as_judge` for backwards compatibility. New clients should send- `type` explicitly.+ **Cloud-only integration deprecation gate (effective 2026-06-22):**+ On Langfuse Cloud, legacy export sources are only accepted for blob+ storage integrations created before 2026-06-22, regardless of+ project age. Requests that would create a new integration with+ `LEGACY_TRACES_OBSERVATIONS` or+ `LEGACY_TRACES_AND_ENRICHED_OBSERVATIONS` are rejected with HTTP+ 400. Use `OBSERVATIONS_V2` instead. Self-hosted deployments are+ unaffected.+ exportFieldGroups:+ type: array+ items:+ $ref: '#/components/schemas/BlobStorageExportFieldGroup'+ nullable: true+ description: >-+ Field groups to include in each exported observation row. Applies to+ all export sources; must include `core` if provided. When omitted on+ create, the column default (all groups) applies. When omitted on+ update, the existing value is preserved.- Unstable API note:- - This surface may evolve while the underlying evaluation data model is- being redesigned.- operationId: unstable_evaluators_create- tags:- - UnstableEvaluators- parameters: []- responses:- '200':- description: ''- content:- application/json:- schema:- $ref: '#/components/schemas/unstableEvaluator'- '400':- description: ''- content:- application/json:- schema: {}- '401':- description: ''- content:- application/json:- schema: {}- '403':- description: ''- content:- application/json:- schema: {}- '404':- description: ''- content:- application/json:- schema: {}- '405':- description: ''- content:- application/json:- schema: {}- '409':- description: ''- content:- application/json:- schema:- $ref: '#/components/schemas/unstablePublicApiError'- '422':- description: ''- content:- application/json:- schema:- $ref: '#/components/schemas/unstablePublicApiError'- '429':- description: ''- content:- application/json:- schema:- $ref: '#/components/schemas/unstablePublicApiError'- '500':- description: ''- content:- application/json:- schema:- $ref: '#/components/schemas/unstablePublicApiError'- security: *ref_0- requestBody:- required: true- content:- application/json:- schema:- $ref: '#/components/schemas/unstableCreateEvaluatorRequest'- get:- description: |-- List the evaluators available to the authenticated project.+ `exportFieldGroups` requires `exportSource` to be provided in the+ same request.+ required:+ - projectId+ - type+ - bucketName+ - region+ - exportFrequency+ - enabled+ - forcePathStyle+ - fileType+ - exportMode+ BlobStorageIntegrationResponse:+ title: BlobStorageIntegrationResponse+ type: object+ properties:+ id:+ type: string+ projectId:+ type: string+ type:+ $ref: '#/components/schemas/BlobStorageIntegrationType'+ bucketName:+ type: string+ endpoint:+ type: string+ nullable: true+ region:+ type: string+ accessKeyId:+ type: string+ nullable: true+ prefix:+ type: string+ exportFrequency:+ $ref: '#/components/schemas/BlobStorageExportFrequency'+ enabled:+ type: boolean+ forcePathStyle:+ type: boolean+ fileType:+ $ref: '#/components/schemas/BlobStorageIntegrationFileTypeResponse'+ exportMode:+ $ref: '#/components/schemas/BlobStorageExportMode'+ exportStartDate:+ type: string+ format: date-time+ nullable: true+ compressed:+ type: boolean+ exportSource:+ $ref: '#/components/schemas/BlobStorageExportSource'+ exportFieldGroups:+ type: array+ items:+ $ref: '#/components/schemas/BlobStorageExportFieldGroup'+ nullable: true+ description: >-+ Field groups included in each exported observation row. An empty+ list is treated as all groups during export.+ nextSyncAt:+ type: string+ format: date-time+ nullable: true+ lastSyncAt:+ type: string+ format: date-time+ nullable: true+ lastError:+ type: string+ nullable: true+ lastErrorAt:+ type: string+ format: date-time+ nullable: true+ createdAt:+ type: string+ format: date-time+ updatedAt:+ type: string+ format: date-time+ required:+ - id+ - projectId+ - type+ - bucketName+ - endpoint+ - region+ - accessKeyId+ - prefix+ - exportFrequency+ - enabled+ - forcePathStyle+ - fileType+ - exportMode+ - exportStartDate+ - compressed+ - exportSource+ - exportFieldGroups+ - nextSyncAt+ - lastSyncAt+ - lastError+ - lastErrorAt+ - createdAt+ - updatedAt+ BlobStorageIntegrationsResponse:+ title: BlobStorageIntegrationsResponse+ type: object+ properties:+ data:+ type: array+ items:+ $ref: '#/components/schemas/BlobStorageIntegrationResponse'+ required:+ - data+ BlobStorageSyncStatus:+ title: BlobStorageSyncStatus+ type: string+ enum:+ - idle+ - running+ - queued+ - up_to_date+ - disabled+ - error+ description: >-+ Sync status of the blob storage integration:- Important behavior:- - This endpoint returns the latest version of each available evaluator.- - Every evaluator is owned by the authenticated project.- operationId: unstable_evaluators_list- tags:- - UnstableEvaluators- parameters:- - name: page- in: query- description: 1-based page number. Defaults to `1`.- required: false- schema:- type: integer- nullable: true- - name: limit- in: query- description: Maximum number of items per page. Defaults to `50`.- required: false- schema:+ - `disabled` — integration is not enabled++ - `error` — last export failed (see `lastError` for details)++ - `running` — an export job is currently being processed++ - `queued` — next export is overdue (`nextSyncAt` is in the past) and+ waiting to be picked up by the worker++ - `idle` — enabled but has never exported yet and no export is queued++ - `up_to_date` — all available data has been exported; next export is+ scheduled for the future+++ **ETL usage**: poll this endpoint and check for `up_to_date` status.+ Compare `lastSyncAt` against your++ ETL bookmark to determine if new data is available. Note that exports+ run with a 20-minute lag buffer,++ so `lastSyncAt` will always be at least 20 minutes behind real-time.+ BlobStorageIntegrationStatusResponse:+ title: BlobStorageIntegrationStatusResponse+ type: object+ properties:+ id:+ type: string+ projectId:+ type: string+ syncStatus:+ $ref: '#/components/schemas/BlobStorageSyncStatus'+ enabled:+ type: boolean+ lastSyncAt:+ type: string+ format: date-time+ nullable: true+ description: >-+ End of the last successfully exported time window. Compare against+ your ETL bookmark to determine if new data is available. Null if the+ integration has never synced.+ nextSyncAt:+ type: string+ format: date-time+ nullable: true+ description: When the next export is scheduled. Null if no sync has occurred yet.+ lastError:+ type: string+ nullable: true+ description: >-+ Raw error message from the storage provider (S3/Azure/GCS) if the+ last export failed. Cleared on successful export.+ lastErrorAt:+ type: string+ format: date-time+ nullable: true+ description: When the last error occurred. Cleared on successful export.+ required:+ - id+ - projectId+ - syncStatus+ - enabled+ - lastSyncAt+ - nextSyncAt+ - lastError+ - lastErrorAt+ BlobStorageIntegrationDeletionResponse:+ title: BlobStorageIntegrationDeletionResponse+ type: object+ properties:+ message:+ type: string+ required:+ - message+ CreateCommentRequest:+ title: CreateCommentRequest+ type: object+ properties:+ projectId:+ type: string+ description: The id of the project to attach the comment to.+ objectType:+ type: string+ description: >-+ The type of the object to attach the comment to (trace, observation,+ session, prompt).+ objectId:+ type: string+ description: >-+ The id of the object to attach the comment to. If this does not+ reference a valid existing object, an error will be thrown.+ content:+ type: string+ description: >-+ The content of the comment. May include markdown. Currently limited+ to 5000 characters.+ authorUserId:+ type: string+ nullable: true+ description: >-+ The id of the user who created the comment. Must be a member of the+ organization that owns the project, otherwise an error will be+ thrown.+ required:+ - projectId+ - objectType+ - objectId+ - content+ CreateCommentResponse:+ title: CreateCommentResponse+ type: object+ properties:+ id:+ type: string+ description: The id of the created object in Langfuse+ required:+ - id+ GetCommentsResponse:+ title: GetCommentsResponse+ type: object+ properties:+ data:+ type: array+ items:+ $ref: '#/components/schemas/Comment'+ meta:+ $ref: '#/components/schemas/utilsMetaResponse'+ required:+ - data+ - meta+ Deprecation:+ title: Deprecation+ type: object+ description: >-+ Migration signal returned by deprecated endpoints. Optional fields are+ omitted when they have no value.+ properties:+ message:+ type: string+ description: >-+ Human- and agent-readable summary of the deprecation and its+ replacement.+ replacement:+ type: string+ nullable: true+ description: >-+ The replacement endpoint, e.g. "GET /api/public/v2/observations".+ Omitted when the endpoint is being removed without a direct+ replacement.+ docsUrl:+ type: string+ nullable: true+ description: Link to the migration documentation (markdown), when available.+ sunsetAt:+ type: string+ nullable: true+ description: >-+ ISO date after which the endpoint may stop working, when a removal+ date is committed.+ required:+ - message+ Trace:+ title: Trace+ type: object+ properties:+ id:+ type: string+ description: The unique identifier of a trace+ timestamp:+ type: string+ format: date-time+ description: The timestamp when the trace was created+ name:+ type: string+ nullable: true+ description: The name of the trace+ input:+ nullable: true+ description: The input data of the trace. Can be any JSON.+ output:+ nullable: true+ description: The output data of the trace. Can be any JSON.+ sessionId:+ type: string+ nullable: true+ description: The session identifier associated with the trace+ release:+ type: string+ nullable: true+ description: The release version of the application when the trace was created+ version:+ type: string+ nullable: true+ description: The version of the trace+ userId:+ type: string+ nullable: true+ description: The user identifier associated with the trace+ metadata:+ nullable: true+ description: The metadata associated with the trace. Can be any JSON.+ tags:+ type: array+ items:+ type: string+ description: The tags associated with the trace.+ public:+ type: boolean+ description: Public traces are accessible via url without login+ environment:+ type: string+ description: >-+ The environment from which this trace originated. Can be any+ lowercase alphanumeric string with hyphens and underscores that does+ not start with 'langfuse'.+ required:+ - id+ - timestamp+ - name+ - sessionId+ - release+ - version+ - userId+ - tags+ - public+ - environment+ TraceWithDetails:+ title: TraceWithDetails+ type: object+ properties:+ htmlPath:+ type: string+ description: Path of trace in Langfuse UI+ latency:+ type: number+ format: double+ nullable: true+ description: Latency of trace in seconds+ totalCost:+ type: number+ format: double+ nullable: true+ description: Cost of trace in USD+ observations:+ type: array+ items:+ type: string+ nullable: true+ description: List of observation ids+ scores:+ type: array+ items:+ type: string+ nullable: true+ description: List of score ids+ required:+ - htmlPath+ allOf:+ - $ref: '#/components/schemas/Trace'+ TraceWithFullDetails:+ title: TraceWithFullDetails+ type: object+ properties:+ htmlPath:+ type: string+ description: Path of trace in Langfuse UI+ latency:+ type: number+ format: double+ nullable: true+ description: Latency of trace in seconds+ totalCost:+ type: number+ format: double+ nullable: true+ description: Cost of trace in USD+ observations:+ type: array+ items:+ $ref: '#/components/schemas/ObservationsView'+ description: List of observations+ scores:+ type: array+ items:+ $ref: '#/components/schemas/ScoreV1'+ description: List of scores+ _deprecation:+ $ref: '#/components/schemas/Deprecation'+ nullable: true+ required:+ - htmlPath+ - observations+ - scores+ allOf:+ - $ref: '#/components/schemas/Trace'+ Session:+ title: Session+ type: object+ properties:+ id:+ type: string+ createdAt:+ type: string+ format: date-time+ projectId:+ type: string+ environment:+ type: string+ description: The environment from which this session originated.+ required:+ - id+ - createdAt+ - projectId+ - environment+ SessionWithTraces:+ title: SessionWithTraces+ type: object+ properties:+ traces:+ type: array+ items:+ $ref: '#/components/schemas/Trace'+ _deprecation:+ $ref: '#/components/schemas/Deprecation'+ nullable: true+ required:+ - traces+ allOf:+ - $ref: '#/components/schemas/Session'+ Observation:+ title: Observation+ type: object+ properties:+ id:+ type: string+ description: The unique identifier of the observation+ traceId:+ type: string+ nullable: true+ description: The trace ID associated with the observation+ type:+ type: string+ description: The type of the observation+ name:+ type: string+ nullable: true+ description: The name of the observation+ startTime:+ type: string+ format: date-time+ description: The start time of the observation+ endTime:+ type: string+ format: date-time+ nullable: true+ description: The end time of the observation.+ completionStartTime:+ type: string+ format: date-time+ nullable: true+ description: The completion start time of the observation+ model:+ type: string+ nullable: true+ description: The model used for the observation+ modelParameters:+ description: The parameters of the model used for the observation+ input:+ description: The input data of the observation+ version:+ type: string+ nullable: true+ description: The version of the observation+ metadata:+ description: Additional metadata of the observation+ output:+ description: The output data of the observation+ usage:+ $ref: '#/components/schemas/Usage'+ description: >-+ (Deprecated. Use usageDetails and costDetails instead.) The usage+ data of the observation+ level:+ $ref: '#/components/schemas/ObservationLevel'+ description: The level of the observation+ statusMessage:+ type: string+ nullable: true+ description: The status message of the observation+ parentObservationId:+ type: string+ nullable: true+ description: The parent observation ID+ promptId:+ type: string+ nullable: true+ description: The prompt ID associated with the observation+ usageDetails:+ type: object+ additionalProperties:type: integer- nullable: true- responses:- '200':- description: ''- content:- application/json:- schema:- $ref: '#/components/schemas/unstableEvaluators'- '400':- description: ''- content:- application/json:- schema: {}- '401':- description: ''- content:- application/json:- schema: {}- '403':- description: ''- content:- application/json:- schema: {}- '404':- description: ''- content:- application/json:- schema: {}- '405':- description: ''- content:- application/json:- schema: {}- '429':- description: ''- content:- application/json:- schema:- $ref: '#/components/schemas/unstablePublicApiError'- '500':- description: ''- content:- application/json:- schema:- $ref: '#/components/schemas/unstablePublicApiError'- security: *ref_0- /api/public/unstable/evaluators/{evaluatorId}:- get:+ description: >-+ The usage details of the observation. Key is the name of the usage+ metric, value is the number of units consumed. The total key is the+ sum of all (non-total) usage metrics or the total value ingested.+ costDetails:+ type: object+ additionalProperties:+ type: number+ format: double+ description: >-+ The cost details of the observation. Key is the name of the cost+ metric, value is the cost in USD. The total key is the sum of all+ (non-total) cost metrics or the total value ingested.+ environment:+ type: string+ description: >-+ The environment from which this observation originated. Can be any+ lowercase alphanumeric string with hyphens and underscores that does+ not start with 'langfuse'.+ required:+ - id+ - traceId+ - type+ - name+ - startTime+ - endTime+ - completionStartTime+ - model+ - modelParameters+ - input+ - version+ - metadata+ - output+ - usage+ - level+ - statusMessage+ - parentObservationId+ - promptId+ - usageDetails+ - costDetails+ - environment+ ObservationsView:+ title: ObservationsView+ type: object+ properties:+ promptName:+ type: string+ nullable: true+ description: The name of the prompt associated with the observation+ promptVersion:+ type: integer+ nullable: true+ description: The version of the prompt associated with the observation+ modelId:+ type: string+ nullable: true+ description: The unique identifier of the model+ inputPrice:+ type: number+ format: double+ nullable: true+ description: The price of the input in USD+ outputPrice:+ type: number+ format: double+ nullable: true+ description: The price of the output in USD.+ totalPrice:+ type: number+ format: double+ nullable: true+ description: The total price in USD.+ calculatedInputCost:+ type: number+ format: double+ nullable: true+ description: >-+ (Deprecated. Use usageDetails and costDetails instead.) The+ calculated cost of the input in USD+ calculatedOutputCost:+ type: number+ format: double+ nullable: true+ description: >-+ (Deprecated. Use usageDetails and costDetails instead.) The+ calculated cost of the output in USD+ calculatedTotalCost:+ type: number+ format: double+ nullable: true+ description: >-+ (Deprecated. Use usageDetails and costDetails instead.) The+ calculated total cost in USD+ latency:+ type: number+ format: double+ nullable: true+ description: The latency in seconds.+ timeToFirstToken:+ type: number+ format: double+ nullable: true+ description: The time to the first token in seconds+ required:+ - promptName+ - promptVersion+ - modelId+ - inputPrice+ - outputPrice+ - totalPrice+ - calculatedInputCost+ - calculatedOutputCost+ - calculatedTotalCost+ - latency+ - timeToFirstToken+ allOf:+ - $ref: '#/components/schemas/Observation'+ ObservationsViewSingle:+ title: ObservationsViewSingle+ type: object+ properties:+ _deprecation:+ $ref: '#/components/schemas/Deprecation'+ nullable: true+ allOf:+ - $ref: '#/components/schemas/ObservationsView'+ ObservationV2:+ title: ObservationV2+ type: objectdescription: >-- Get one evaluator by `id`.+ An observation from the v2 API with field-group-based selection.++ Core fields are always present. Other fields are included only when+ their field group is requested.+ properties:+ id:+ type: string+ description: The unique identifier of the observation+ traceId:+ type: string+ nullable: true+ description: The trace ID associated with the observation+ startTime:+ type: string+ format: date-time+ description: The start time of the observation+ endTime:+ type: string+ format: date-time+ nullable: true+ description: The end time of the observation+ projectId:+ type: string+ description: The project ID this observation belongs to+ parentObservationId:+ type: string+ nullable: true+ description: >-+ The physical parent observation ID, if present.+ Observations marked as app roots by the SDK may retain a non-null+ parent ID.+ type:+ type: string+ description: The type of the observation (e.g. GENERATION, SPAN, EVENT)+ isRootObservation:+ type: boolean+ nullable: true+ description: >-+ Whether this observation is a logical root.- This endpoint always returns the evaluator's latest version. Use it when- you want the current prompt, output definition, model configuration, and- derived variables for the evaluator you plan to use in an evaluation- rule.- operationId: unstable_evaluators_get- tags:- - UnstableEvaluators- parameters:- - name: evaluatorId- in: path- description: Evaluator identifier returned by the evaluator endpoints.- required: true- schema:+ This is true for observations without a physical parent and+ observations marked as app roots by the SDK.+ name:+ type: string+ nullable: true+ description: The name of the observation+ level:+ $ref: '#/components/schemas/ObservationLevel'+ nullable: true+ description: The level of the observation+ statusMessage:+ type: string+ nullable: true+ description: The status message of the observation+ version:+ type: string+ nullable: true+ description: The version of the observation+ environment:+ type: string+ nullable: true+ description: The environment from which this observation originated+ bookmarked:+ type: boolean+ nullable: true+ description: Whether the observation is bookmarked+ public:+ type: boolean+ nullable: true+ description: Whether the observation is public+ userId:+ type: string+ nullable: true+ description: The user ID associated with the observation+ sessionId:+ type: string+ nullable: true+ description: The session ID associated with the observation+ completionStartTime:+ type: string+ format: date-time+ nullable: true+ description: The completion start time of the observation+ createdAt:+ type: string+ format: date-time+ nullable: true+ description: The creation timestamp of the observation+ updatedAt:+ type: string+ format: date-time+ nullable: true+ description: The last update timestamp of the observation+ input:+ nullable: true+ description: The input data of the observation+ output:+ nullable: true+ description: The output data of the observation+ metadata:+ nullable: true+ description: Additional metadata of the observation+ providedModelName:+ type: string+ nullable: true+ description: The model name as provided by the user+ internalModelId:+ type: string+ nullable: true+ description: The internal model ID matched by Langfuse+ modelParameters:+ nullable: true+ description: The parameters of the model used for the observation+ usageDetails:+ type: object+ additionalProperties:+ type: integer+ nullable: true+ description: >-+ The usage details of the observation. Key is the usage metric name,+ value is the number of units consumed.+ costDetails:+ type: object+ additionalProperties:+ type: number+ format: double+ nullable: true+ description: >-+ The cost details of the observation. Key is the cost metric name,+ value is the cost in USD.+ totalCost:+ type: number+ format: double+ nullable: true+ description: The total cost of the observation in USD+ usagePricingTierName:+ type: string+ nullable: true+ description: >-+ The name of the pricing tier applied to this observation's usage+ costs+ promptId:+ type: string+ nullable: true+ description: The prompt ID associated with the observation+ promptName:+ type: string+ nullable: true+ description: The prompt name associated with the observation+ promptVersion:+ type: integer+ nullable: true+ description: The prompt version associated with the observation+ latency:+ type: number+ format: double+ nullable: true+ description: The latency in seconds+ timeToFirstToken:+ type: number+ format: double+ nullable: true+ description: The time to first token in seconds+ modelId:+ type: string+ nullable: true+ description: >-+ The matched model ID. Null when the `model` field group is not+ requested.+ inputPrice:+ type: string+ nullable: true+ description: >-+ The input token price (USD per unit) from the matched model,+ serialized as a decimal string (e.g. "0.0001"). Null when the+ `model` field group is not requested.+ outputPrice:+ type: string+ nullable: true+ description: >-+ The output token price (USD per unit) from the matched model,+ serialized as a decimal string (e.g. "0.0001"). Null when the+ `model` field group is not requested.+ totalPrice:+ type: string+ nullable: true+ description: >-+ The total token price (USD per unit) from the matched model,+ serialized as a decimal string (e.g. "0.0001"). Null when the+ `model` field group is not requested.+ traceName:+ type: string+ nullable: true+ description: The name of the parent trace+ tags:+ type: array+ items:type: string- responses:- '200':- description: ''- content:- application/json:- schema:- $ref: '#/components/schemas/unstableEvaluator'- '400':- description: ''- content:- application/json:- schema: {}- '401':- description: ''- content:- application/json:- schema: {}- '403':- description: ''- content:- application/json:- schema: {}- '404':- description: ''- content:- application/json:- schema: {}- '405':- description: ''- content:- application/json:- schema: {}- '429':- description: ''- content:- application/json:- schema:- $ref: '#/components/schemas/unstablePublicApiError'- '500':- description: ''- content:- application/json:- schema:- $ref: '#/components/schemas/unstablePublicApiError'- security: *ref_0- delete:+ nullable: true+ description: Tags from the parent trace (denormalized onto the observation)+ release:+ type: string+ nullable: true+ description: The release version of the parent trace+ required:+ - id+ - traceId+ - startTime+ - endTime+ - projectId+ - parentObservationId+ - type+ - modelId+ - inputPrice+ - outputPrice+ - totalPrice+ Usage:+ title: Usage+ type: objectdescription: >-- Delete an evaluator.--- Important behavior:-- - This deletes the evaluator including all of its stored versions.-- - Evaluation rule assignments referencing the evaluator are also- deleted.-- - Scores already produced by the evaluator are not deleted.- operationId: unstable_evaluators_delete- tags:- - UnstableEvaluators- parameters:- - name: evaluatorId- in: path- description: Evaluator identifier returned by the evaluator endpoints.- required: true- schema:- type: string- responses:- '200':- description: ''- content:- application/json:- schema:- $ref: '#/components/schemas/unstableDeleteEvaluatorResponse'- '400':- description: ''- content:- application/json:- schema: {}- '401':- description: ''- content:- application/json:- schema: {}- '403':- description: ''- content:- application/json:- schema: {}- '404':- description: ''- content:- application/json:- schema: {}- '405':- description: ''- content:- application/json:- schema: {}- '429':- description: ''- content:- application/json:- schema:- $ref: '#/components/schemas/unstablePublicApiError'- '500':- description: ''- content:- application/json:- schema:- $ref: '#/components/schemas/unstablePublicApiError'- security: *ref_0-components:- schemas:- AnnotationQueueStatus:- title: AnnotationQueueStatus- type: string- enum:- - PENDING- - COMPLETED- AnnotationQueueObjectType:- title: AnnotationQueueObjectType- type: string- enum:- - TRACE- - OBSERVATION- - SESSION- AnnotationQueue:- title: AnnotationQueue+ (Deprecated. Use usageDetails and costDetails instead.) Standard+ interface for usage and cost+ properties:+ input:+ type: integer+ description: Number of input units (e.g. tokens)+ output:+ type: integer+ description: Number of output units (e.g. tokens)+ total:+ type: integer+ description: Defaults to input+output if not set+ unit:+ type: string+ nullable: true+ description: Unit of measurement+ inputCost:+ type: number+ format: double+ nullable: true+ description: USD input cost+ outputCost:+ type: number+ format: double+ nullable: true+ description: USD output cost+ totalCost:+ type: number+ format: double+ nullable: true+ description: USD total cost, defaults to input+output+ required:+ - input+ - output+ - total+ - unit+ ScoreConfig:+ title: ScoreConfigtype: object+ description: Configuration for a scoreproperties:id:type: stringname:type: string- description:+ createdAt:type: string+ format: date-time+ updatedAt:+ type: string+ format: date-time+ projectId:+ type: string+ dataType:+ $ref: '#/components/schemas/ScoreConfigDataType'+ isArchived:+ type: boolean+ description: Whether the score config is archived. Defaults to false+ minValue:+ type: number+ format: doublenullable: true- scoreConfigIds:+ description: >-+ Sets minimum value for numerical scores. If not set, the minimum+ value defaults to -∞+ maxValue:+ type: number+ format: double+ nullable: true+ description: >-+ Sets maximum value for numerical scores. If not set, the maximum+ value defaults to +∞+ categories:type: arrayitems:- type: string- createdAt:- type: string- format: date-time- updatedAt:+ $ref: '#/components/schemas/ConfigCategory'+ nullable: true+ description: Configures custom categories for categorical scores+ description:type: string- format: date-time+ nullable: true+ description: Description of the score configrequired:- id- name- - description- - scoreConfigIds- createdAt- updatedAt- AnnotationQueueItem:- title: AnnotationQueueItem+ - projectId+ - dataType+ - isArchived+ ConfigCategory:+ title: ConfigCategory+ type: object+ properties:+ value:+ type: number+ format: double+ label:+ type: string+ required:+ - value+ - label+ BaseScoreV1:+ title: BaseScoreV1type: objectproperties:id:type: string- queueId:+ traceId:type: string- objectId:+ name:type: string- objectType:- $ref: '#/components/schemas/AnnotationQueueObjectType'- status:- $ref: '#/components/schemas/AnnotationQueueStatus'- completedAt:+ source:+ $ref: '#/components/schemas/ScoreSource'+ observationId:type: string- format: date-timenullable: true+ description: The observation ID associated with the score+ timestamp:+ type: string+ format: date-timecreatedAt:type: stringformat: date-timeupdatedAt:type: stringformat: date-time+ authorUserId:+ type: string+ nullable: true+ description: The user ID of the author+ comment:+ type: string+ nullable: true+ description: Comment on the score+ metadata:+ description: Metadata associated with the score+ configId:+ type: string+ nullable: true+ description: >-+ Reference a score config on a score. When set, config and score name+ must be equal and value must comply to optionally defined numerical+ range+ queueId:+ type: string+ nullable: true+ description: >-+ The annotation queue referenced by the score. Indicates if score was+ initially created while processing annotation queue.+ environment:+ type: string+ description: >-+ The environment from which this score originated. Can be any+ lowercase alphanumeric string with hyphens and underscores that does+ not start with 'langfuse'.required:- id- - queueId- - objectId- - objectType- - status+ - traceId+ - name+ - source+ - timestamp- createdAt- updatedAt- PaginatedAnnotationQueues:- title: PaginatedAnnotationQueues+ - authorUserId+ - comment+ - metadata+ - configId+ - queueId+ - environment+ NumericScoreV1:+ title: NumericScoreV1type: objectproperties:- data:- type: array- items:- $ref: '#/components/schemas/AnnotationQueue'- meta:- $ref: '#/components/schemas/utilsMetaResponse'+ value:+ type: number+ format: double+ description: The numeric value of the scorerequired:- - data- - meta- PaginatedAnnotationQueueItems:- title: PaginatedAnnotationQueueItems+ - value+ allOf:+ - $ref: '#/components/schemas/BaseScoreV1'+ BooleanScoreV1:+ title: BooleanScoreV1type: objectproperties:- data:- type: array- items:- $ref: '#/components/schemas/AnnotationQueueItem'- meta:- $ref: '#/components/schemas/utilsMetaResponse'+ value:+ type: number+ format: double+ description: >-+ The numeric value of the score. Equals 1 for "True" and 0 for "False"+ stringValue:+ type: string+ description: >-+ The string representation of the score value. Is inferred from the+ numeric value and equals "True" or "False"required:- - data- - meta- CreateAnnotationQueueRequest:- title: CreateAnnotationQueueRequest+ - value+ - stringValue+ allOf:+ - $ref: '#/components/schemas/BaseScoreV1'+ CategoricalScoreV1:+ title: CategoricalScoreV1+ type: object+ properties:+ value:+ type: number+ format: double+ description: >-+ Represents the numeric category mapping of the stringValue. If no+ config is linked, defaults to 0.+ stringValue:+ type: string+ description: >-+ The string representation of the score value. If no config is+ linked, can be any string. Otherwise, must map to a config category+ required:+ - value+ - stringValue+ allOf:+ - $ref: '#/components/schemas/BaseScoreV1'+ TextScoreV1:+ title: TextScoreV1+ type: object+ properties:+ stringValue:+ type: string+ description: The text content of the score (1-500 characters)+ required:+ - stringValue+ allOf:+ - $ref: '#/components/schemas/BaseScoreV1'+ ScoreV1:+ title: ScoreV1+ oneOf:+ - type: object+ allOf:+ - type: object+ properties:+ dataType:+ type: string+ enum:+ - NUMERIC+ - $ref: '#/components/schemas/NumericScoreV1'+ required:+ - dataType+ - type: object+ allOf:+ - type: object+ properties:+ dataType:+ type: string+ enum:+ - CATEGORICAL+ - $ref: '#/components/schemas/CategoricalScoreV1'+ required:+ - dataType+ - type: object+ allOf:+ - type: object+ properties:+ dataType:+ type: string+ enum:+ - BOOLEAN+ - $ref: '#/components/schemas/BooleanScoreV1'+ required:+ - dataType+ - type: object+ allOf:+ - type: object+ properties:+ dataType:+ type: string+ enum:+ - TEXT+ - $ref: '#/components/schemas/TextScoreV1'+ required:+ - dataType+ BaseScore:+ title: BaseScoretype: objectproperties:+ id:+ type: string+ traceId:+ type: string+ nullable: true+ description: The trace ID associated with the score+ sessionId:+ type: string+ nullable: true+ description: The session ID associated with the score+ observationId:+ type: string+ nullable: true+ description: The observation ID associated with the score+ datasetRunId:+ type: string+ nullable: true+ description: The dataset run ID associated with the scorename:type: string- description:+ source:+ $ref: '#/components/schemas/ScoreSource'+ timestamp:+ type: string+ format: date-time+ createdAt:+ type: string+ format: date-time+ updatedAt:+ type: string+ format: date-time+ authorUserId:type: stringnullable: true- scoreConfigIds:- type: array- items:- type: string- required:- - name- - scoreConfigIds- CreateAnnotationQueueItemRequest:- title: CreateAnnotationQueueItemRequest- type: object- properties:- objectId:+ description: The user ID of the author+ comment:type: string- objectType:- $ref: '#/components/schemas/AnnotationQueueObjectType'- status:- $ref: '#/components/schemas/AnnotationQueueStatus'nullable: true- description: Defaults to PENDING for new queue items- required:- - objectId- - objectType- UpdateAnnotationQueueItemRequest:- title: UpdateAnnotationQueueItemRequest- type: object- properties:- status:- $ref: '#/components/schemas/AnnotationQueueStatus'+ description: Comment on the score+ metadata:+ description: Metadata associated with the score+ configId:+ type: stringnullable: true- DeleteAnnotationQueueItemResponse:- title: DeleteAnnotationQueueItemResponse+ description: >-+ Reference a score config on a score. When set, config and score name+ must be equal and value must comply to optionally defined numerical+ range+ queueId:+ type: string+ nullable: true+ description: >-+ The annotation queue referenced by the score. Indicates if score was+ initially created while processing annotation queue.+ environment:+ type: string+ description: >-+ The environment from which this score originated. Can be any+ lowercase alphanumeric string with hyphens and underscores that does+ not start with 'langfuse'.+ required:+ - id+ - name+ - source+ - timestamp+ - createdAt+ - updatedAt+ - authorUserId+ - comment+ - metadata+ - configId+ - queueId+ - environment+ NumericScore:+ title: NumericScoretype: objectproperties:- success:- type: boolean- message:- type: string+ value:+ type: number+ format: double+ description: The numeric value of the scorerequired:- - success- - message- AnnotationQueueAssignmentRequest:- title: AnnotationQueueAssignmentRequest+ - value+ allOf:+ - $ref: '#/components/schemas/BaseScore'+ BooleanScore:+ title: BooleanScoretype: objectproperties:- userId:+ value:+ type: number+ format: double+ description: >-+ The numeric value of the score. Equals 1 for "True" and 0 for "False"+ stringValue:type: string+ description: >-+ The string representation of the score value. Is inferred from the+ numeric value and equals "True" or "False"required:- - userId- DeleteAnnotationQueueAssignmentResponse:- title: DeleteAnnotationQueueAssignmentResponse+ - value+ - stringValue+ allOf:+ - $ref: '#/components/schemas/BaseScore'+ CategoricalScore:+ title: CategoricalScoretype: objectproperties:- success:- type: boolean+ value:+ type: number+ format: double+ description: >-+ Represents the numeric category mapping of the stringValue. If no+ config is linked, defaults to 0.+ stringValue:+ type: string+ description: >-+ The string representation of the score value. If no config is+ linked, can be any string. Otherwise, must map to a config categoryrequired:- - success- CreateAnnotationQueueAssignmentResponse:- title: CreateAnnotationQueueAssignmentResponse+ - value+ - stringValue+ allOf:+ - $ref: '#/components/schemas/BaseScore'+ CorrectionScore:+ title: CorrectionScoretype: objectproperties:- userId:- type: string- queueId:- type: string- projectId:+ value:+ type: number+ format: double+ description: The numeric value of the score. Always 0 for correction scores.+ stringValue:type: string+ description: The string representation of the correction contentrequired:- - userId- - queueId- - projectId- BlobStorageIntegrationType:- title: BlobStorageIntegrationType- type: string- enum:- - S3- - S3_COMPATIBLE- - AZURE_BLOB_STORAGE- BlobStorageIntegrationFileType:- title: BlobStorageIntegrationFileType- type: string- enum:- - JSON- - CSV- - JSONL- - PARQUET- description: >-- File format for exported data. `PARQUET` is a columnar binary format- encoded and compressed by the storage engine; gzip compression does not- apply to it. Note that the model-price columns (`input_price`,- `output_price`, `total_price`) are not included in Parquet observation- exports.- BlobStorageIntegrationFileTypeResponse:- title: BlobStorageIntegrationFileTypeResponse- type: string- enum:- - JSON- - CSV- - JSONL- - PARQUET- description: File type reported for an existing integration.- BlobStorageExportMode:- title: BlobStorageExportMode- type: string- enum:- - FULL_HISTORY- - FROM_TODAY- - FROM_CUSTOM_DATE- BlobStorageExportFrequency:- title: BlobStorageExportFrequency- type: string- enum:- - every_20_minutes- - hourly- - daily- - weekly- BlobStorageExportSource:- title: BlobStorageExportSource- type: string- enum:- - LEGACY_TRACES_OBSERVATIONS- - OBSERVATIONS_V2- - LEGACY_TRACES_AND_ENRICHED_OBSERVATIONS- description: >-- What data the integration exports.-- - `LEGACY_TRACES_OBSERVATIONS`: traces, observations, and scores tables.- Observation columns are controlled by `exportFieldGroups`; field groups- without a counterpart in this data model (e.g. `trace_context`) are- omitted.-- - `OBSERVATIONS_V2`: same data model as the- `/api/public/v2/observations` endpoint, plus scores. Columns are- controlled by `exportFieldGroups`.-- - `LEGACY_TRACES_AND_ENRICHED_OBSERVATIONS`: both sets. Observation- columns of both portions are controlled by `exportFieldGroups`.--- **Note:** which sources a deployment accepts depends on how far it has- moved to the v4 data model. `OBSERVATIONS_V2` and the- enriched-observations portion of- `LEGACY_TRACES_AND_ENRICHED_OBSERVATIONS` read the enriched observations- table, so they require a deployment that already populates it.- `LEGACY_TRACES_OBSERVATIONS` and the legacy portion of- `LEGACY_TRACES_AND_ENRICHED_OBSERVATIONS` read the legacy traces and- observations tables, so they require a deployment that still populates- those. A deployment part-way through the migration populates both and- accepts every source. Selecting a source the deployment cannot serve is- rejected with `400`, rather than exporting an empty result. See- https://langfuse.com/docs/v4.- BlobStorageExportFieldGroup:- title: BlobStorageExportFieldGroup- type: string- enum:- - core- - basic- - time- - io- - metadata- - model- - usage- - prompt- - metrics- - tools- - trace_context- description: >-- Field group selecting which observation columns are included in the- export. Applies to all export sources; groups without a counterpart in- the legacy data model (e.g. `trace_context`) are omitted from the legacy- observations export.- CreateBlobStorageIntegrationRequest:- title: CreateBlobStorageIntegrationRequest+ - value+ - stringValue+ allOf:+ - $ref: '#/components/schemas/BaseScore'+ TextScore:+ title: TextScoretype: objectproperties:- projectId:+ stringValue:type: string- description: ID of the project in which to configure the blob storage integration- type:- $ref: '#/components/schemas/BlobStorageIntegrationType'- bucketName:+ description: The text content of the score (1-500 characters)+ required:+ - stringValue+ allOf:+ - $ref: '#/components/schemas/BaseScore'+ Score:+ title: Score+ oneOf:+ - type: object+ allOf:+ - type: object+ properties:+ dataType:+ type: string+ enum:+ - NUMERIC+ - $ref: '#/components/schemas/NumericScore'+ required:+ - dataType+ - type: object+ allOf:+ - type: object+ properties:+ dataType:+ type: string+ enum:+ - CATEGORICAL+ - $ref: '#/components/schemas/CategoricalScore'+ required:+ - dataType+ - type: object+ allOf:+ - type: object+ properties:+ dataType:+ type: string+ enum:+ - BOOLEAN+ - $ref: '#/components/schemas/BooleanScore'+ required:+ - dataType+ - type: object+ allOf:+ - type: object+ properties:+ dataType:+ type: string+ enum:+ - CORRECTION+ - $ref: '#/components/schemas/CorrectionScore'+ required:+ - dataType+ - type: object+ allOf:+ - type: object+ properties:+ dataType:+ type: string+ enum:+ - TEXT+ - $ref: '#/components/schemas/TextScore'+ required:+ - dataType+ properties:+ _deprecation:+ $ref: '#/components/schemas/Deprecation'+ nullable: true+ CreateScoreValue:+ title: CreateScoreValue+ oneOf:+ - type: number+ format: double+ - type: string+ description: >-+ The value of the score. Must be passed as string for categorical and+ text scores, and numeric for boolean and numeric scores+ Comment:+ title: Comment+ type: object+ properties:+ id:type: string- description: >-- Name of the storage bucket. For AZURE_BLOB_STORAGE, must be a valid- Azure container name (3-63 chars, lowercase letters, numbers, and- hyphens only, must start and end with a letter or number, no- consecutive hyphens).- endpoint:+ projectId:type: string- nullable: true- description: Custom endpoint URL (required for S3_COMPATIBLE type)- region:+ createdAt:type: string- description: Storage region- accessKeyId:+ format: date-time+ updatedAt:type: string- nullable: true- description: Access key ID for authentication- secretAccessKey:+ format: date-time+ objectType:+ $ref: '#/components/schemas/CommentObjectType'+ objectId:type: string- nullable: true- description: Secret access key for authentication (will be encrypted when stored)- prefix:+ content:type: string- nullable: true- description: >-- Path prefix for exported files (must end with forward slash if- provided)- exportFrequency:- $ref: '#/components/schemas/BlobStorageExportFrequency'- enabled:- type: boolean- description: Whether the integration is active- forcePathStyle:- type: boolean- description: Use path-style URLs for S3 requests- fileType:- $ref: '#/components/schemas/BlobStorageIntegrationFileType'- exportMode:- $ref: '#/components/schemas/BlobStorageExportMode'- exportStartDate:+ authorUserId:type: string- format: date-time- nullable: true- description: >-- Custom start date for exports (required when exportMode is- FROM_CUSTOM_DATE). Must not be in the future (27 h tolerance for- timezone differences).- compressed:- type: boolean- nullable: true- description: >-- Enable gzip compression for exported files (.csv.gz, .json.gz,- .jsonl.gz). Defaults to true.- exportSource:- $ref: '#/components/schemas/BlobStorageExportSource'- nullable: true- description: >-- Data to export. When omitted on update, the existing value is- preserved. When omitted on create, the default is `OBSERVATIONS_V2`- on Langfuse Cloud, and on self-hosted deployments- `LEGACY_TRACES_OBSERVATIONS` — or `OBSERVATIONS_V2` where the- deployment no longer populates the legacy tables. The default is- never a source the deployment cannot serve. Required when- `exportFieldGroups` is provided.--- **Cloud-only project deprecation gate (effective 2026-05-20):** For- projects created on or after 2026-05-20 on Langfuse Cloud,- `LEGACY_TRACES_OBSERVATIONS` and- `LEGACY_TRACES_AND_ENRICHED_OBSERVATIONS` are rejected with HTTP- 400. Use `OBSERVATIONS_V2` for all new integrations. Self-hosted- deployments are unaffected.--- **Cloud-only integration deprecation gate (effective 2026-06-22):**- On Langfuse Cloud, legacy export sources are only accepted for blob- storage integrations created before 2026-06-22, regardless of- project age. Requests that would create a new integration with- `LEGACY_TRACES_OBSERVATIONS` or- `LEGACY_TRACES_AND_ENRICHED_OBSERVATIONS` are rejected with HTTP- 400. Use `OBSERVATIONS_V2` instead. Self-hosted deployments are- unaffected.- exportFieldGroups:- type: array- items:- $ref: '#/components/schemas/BlobStorageExportFieldGroup'nullable: true- description: >-- Field groups to include in each exported observation row. Applies to- all export sources; must include `core` if provided. When omitted on- create, the column default (all groups) applies. When omitted on- update, the existing value is preserved.--- `exportFieldGroups` requires `exportSource` to be provided in the- same request.+ description: The user ID of the comment authorrequired:+ - id- projectId- - type- - bucketName- - region- - exportFrequency- - enabled- - forcePathStyle- - fileType- - exportMode- BlobStorageIntegrationResponse:- title: BlobStorageIntegrationResponse+ - createdAt+ - updatedAt+ - objectType+ - objectId+ - content+ Dataset:+ title: Datasettype: objectproperties:id:type: string- projectId:- type: string- type:- $ref: '#/components/schemas/BlobStorageIntegrationType'- bucketName:+ name:type: string- endpoint:+ description:type: stringnullable: true- region:- type: string- accessKeyId:- type: string+ description: Description of the dataset+ metadata:+ description: Metadata associated with the dataset+ inputSchema:nullable: true- prefix:+ description: JSON Schema for validating dataset item inputs+ expectedOutputSchema:+ nullable: true+ description: JSON Schema for validating dataset item expected outputs+ projectId:type: string- exportFrequency:- $ref: '#/components/schemas/BlobStorageExportFrequency'- enabled:- type: boolean- forcePathStyle:- type: boolean- fileType:- $ref: '#/components/schemas/BlobStorageIntegrationFileTypeResponse'- exportMode:- $ref: '#/components/schemas/BlobStorageExportMode'- exportStartDate:+ createdAt:type: stringformat: date-time- nullable: true- compressed:- type: boolean- exportSource:- $ref: '#/components/schemas/BlobStorageExportSource'- exportFieldGroups:- type: array- items:- $ref: '#/components/schemas/BlobStorageExportFieldGroup'- nullable: true- description: >-- Field groups included in each exported observation row. An empty- list is treated as all groups during export.- nextSyncAt:+ updatedAt:type: stringformat: date-time- nullable: true- lastSyncAt:+ required:+ - id+ - name+ - description+ - metadata+ - inputSchema+ - expectedOutputSchema+ - projectId+ - createdAt+ - updatedAt+ DatasetItem:+ title: DatasetItem+ type: object+ properties:+ id:type: string- format: date-time- nullable: true- lastError:+ status:+ $ref: '#/components/schemas/DatasetStatus'+ input:+ description: Input data for the dataset item+ expectedOutput:+ description: Expected output for the dataset item+ metadata:+ description: Metadata associated with the dataset item+ sourceTraceId:type: stringnullable: true- lastErrorAt:+ description: The trace ID that sourced this dataset item+ sourceObservationId:type: string- format: date-timenullable: true+ description: The observation ID that sourced this dataset item+ datasetId:+ type: string+ datasetName:+ type: stringcreatedAt:type: stringformat: date-timeupdatedAt:type: stringformat: date-time+ mediaReferences:+ type: array+ items:+ $ref: '#/components/schemas/DatasetItemMediaReference'+ description: >-+ Resolved Langfuse media references found in input, expectedOutput,+ and metadata.required:- id- - projectId- - type- - bucketName- - endpoint- - region- - accessKeyId- - prefix- - exportFrequency- - enabled- - forcePathStyle- - fileType- - exportMode- - exportStartDate- - compressed- - exportSource- - exportFieldGroups- - nextSyncAt- - lastSyncAt- - lastError- - lastErrorAt+ - status+ - input+ - expectedOutput+ - metadata+ - sourceTraceId+ - sourceObservationId+ - datasetId+ - datasetName- createdAt- updatedAt- BlobStorageIntegrationsResponse:- title: BlobStorageIntegrationsResponse+ - mediaReferences+ DatasetItemMediaReference:+ title: DatasetItemMediaReferencetype: objectproperties:- data:- type: array- items:- $ref: '#/components/schemas/BlobStorageIntegrationResponse'+ field:+ $ref: '#/components/schemas/DatasetItemMediaReferenceField'+ description: The dataset item field containing the reference+ referenceString:+ type: string+ description: >-+ The Langfuse media reference string, e.g.+ `@@@langfuseMedia:type=image/png|id=...|source=bytes@@@`+ jsonPath:+ type: string+ description: >-+ JSONPath of the string holding the reference within the field, e.g.+ `$['image']`+ media:+ $ref: '#/components/schemas/DatasetItemMediaReferenceMedia'+ description: The resolved media record.required:- - data- BlobStorageSyncStatus:- title: BlobStorageSyncStatus+ - field+ - referenceString+ - jsonPath+ - media+ DatasetItemMediaReferenceField:+ title: DatasetItemMediaReferenceFieldtype: stringenum:- - idle- - running- - queued- - up_to_date- - disabled- - error- description: >-- Sync status of the blob storage integration:-- - `disabled` — integration is not enabled-- - `error` — last export failed (see `lastError` for details)-- - `running` — an export job is currently being processed-- - `queued` — next export is overdue (`nextSyncAt` is in the past) and- waiting to be picked up by the worker-- - `idle` — enabled but has never exported yet and no export is queued-- - `up_to_date` — all available data has been exported; next export is- scheduled for the future--- **ETL usage**: poll this endpoint and check for `up_to_date` status.- Compare `lastSyncAt` against your-- ETL bookmark to determine if new data is available. Note that exports- run with a 20-minute lag buffer,-- so `lastSyncAt` will always be at least 20 minutes behind real-time.- BlobStorageIntegrationStatusResponse:- title: BlobStorageIntegrationStatusResponse+ - input+ - expectedOutput+ - metadata+ DatasetItemMediaReferenceMedia:+ title: DatasetItemMediaReferenceMedia+ type: object+ properties:+ mediaId:+ type: string+ description: The unique langfuse identifier of the media record+ contentType:+ type: string+ description: The MIME type of the media record+ contentLength:+ type: integer+ description: The size of the media record in bytes+ url:+ type: string+ description: The signed download URL of the media record+ urlExpiry:+ type: string+ description: The expiry date and time of the download URL+ required:+ - mediaId+ - contentType+ - contentLength+ - url+ - urlExpiry+ DatasetRunItem:+ title: DatasetRunItemtype: objectproperties:id:type: string- projectId:+ datasetRunId:type: string- syncStatus:- $ref: '#/components/schemas/BlobStorageSyncStatus'- enabled:- type: boolean- lastSyncAt:+ datasetRunName:type: string- format: date-time- nullable: true- description: >-- End of the last successfully exported time window. Compare against- your ETL bookmark to determine if new data is available. Null if the- integration has never synced.- nextSyncAt:+ datasetItemId:type: string- format: date-time- nullable: true- description: When the next export is scheduled. Null if no sync has occurred yet.- lastError:+ traceId:+ type: string+ observationId:type: stringnullable: true- description: >-- Raw error message from the storage provider (S3/Azure/GCS) if the- last export failed. Cleared on successful export.- lastErrorAt:+ description: The observation ID associated with this run item+ createdAt:+ type: string+ format: date-time+ updatedAt:type: stringformat: date-time- nullable: true- description: When the last error occurred. Cleared on successful export.required:- id- - projectId- - syncStatus- - enabled- - lastSyncAt- - nextSyncAt- - lastError- - lastErrorAt- BlobStorageIntegrationDeletionResponse:- title: BlobStorageIntegrationDeletionResponse+ - datasetRunId+ - datasetRunName+ - datasetItemId+ - traceId+ - observationId+ - createdAt+ - updatedAt+ DatasetRun:+ title: DatasetRuntype: objectproperties:- message:+ id:type: string- required:- - message- CreateCommentRequest:- title: CreateCommentRequest- type: object- properties:- projectId:+ description: Unique identifier of the dataset run+ name:type: string- description: The id of the project to attach the comment to.- objectType:+ description: Name of the dataset run+ description:type: string- description: >-- The type of the object to attach the comment to (trace, observation,- session, prompt).- objectId:+ nullable: true+ description: Description of the run+ metadata:+ description: Metadata of the dataset run+ datasetId:type: string- description: >-- The id of the object to attach the comment to. If this does not- reference a valid existing object, an error will be thrown.- content:+ description: Id of the associated dataset+ datasetName:type: string- description: >-- The content of the comment. May include markdown. Currently limited- to 5000 characters.- authorUserId:+ description: Name of the associated dataset+ createdAt:type: string- nullable: true- description: >-- The id of the user who created the comment. Must be a member of the- organization that owns the project, otherwise an error will be- thrown.- required:- - projectId- - objectType- - objectId- - content- CreateCommentResponse:- title: CreateCommentResponse- type: object- properties:- id:+ format: date-time+ description: The date and time when the dataset run was created+ updatedAt:type: string- description: The id of the created object in Langfuse+ format: date-time+ description: The date and time when the dataset run was last updatedrequired:- id- GetCommentsResponse:- title: GetCommentsResponse+ - name+ - description+ - metadata+ - datasetId+ - datasetName+ - createdAt+ - updatedAt+ DatasetRunWithItems:+ title: DatasetRunWithItemstype: objectproperties:- data:+ datasetRunItems:type: arrayitems:- $ref: '#/components/schemas/Comment'- meta:- $ref: '#/components/schemas/utilsMetaResponse'+ $ref: '#/components/schemas/DatasetRunItem'+ _deprecation:+ $ref: '#/components/schemas/Deprecation'+ nullable: truerequired:- - data- - meta- Deprecation:- title: Deprecation+ - datasetRunItems+ allOf:+ - $ref: '#/components/schemas/DatasetRun'+ Model:+ title: Modeltype: objectdescription: >-- Migration signal returned by deprecated endpoints. Optional fields are- omitted when they have no value.+ Model definition used for transforming usage into USD cost and/or+ tokenization.+++ Models can have either simple flat pricing or tiered pricing:++ - Flat pricing: Single price per usage type (legacy, but still+ supported)++ - Tiered pricing: Multiple pricing tiers with conditional matching based+ on usage patterns or observation attributes+++ The pricing tiers approach is recommended for models with usage-based+ pricing variations.++ When using tiered pricing, the flat price fields (inputPrice,+ outputPrice, prices) are populated++ from the default tier for backward compatibility.properties:- message:+ id:type: string- description: >-- Human- and agent-readable summary of the deprecation and its- replacement.- replacement:+ modelName:type: string- nullable: truedescription: >-- The replacement endpoint, e.g. "GET /api/public/v2/observations".- Omitted when the endpoint is being removed without a direct- replacement.- docsUrl:- type: string- nullable: true- description: Link to the migration documentation (markdown), when available.- sunsetAt:+ Name of the model definition. If multiple with the same name exist,+ they are applied in the following order: (1) custom over built-in,+ (2) newest according to startTime where+ model.startTime<observation.startTime+ matchPattern:type: string- nullable: truedescription: >-- ISO date after which the endpoint may stop working, when a removal- date is committed.- required:- - message- Trace:- title: Trace- type: object- properties:- id:- type: string- description: The unique identifier of a trace- timestamp:+ Regex pattern which matches this model definition to+ generation.model. Useful in case of fine-tuned models. If you want+ to exact match, use `(?i)^modelname$`+ startDate:type: stringformat: date-time- description: The timestamp when the trace was created- name:- type: stringnullable: true- description: The name of the trace- input:+ description: Apply only to generations which are newer than this ISO date.+ unit:+ $ref: '#/components/schemas/ModelUsageUnit'nullable: true- description: The input data of the trace. Can be any JSON.- output:+ description: Unit used by this model.+ inputPrice:+ type: number+ format: doublenullable: true- description: The output data of the trace. Can be any JSON.- sessionId:- type: string+ description: Deprecated. See 'prices' instead. Price (USD) per input unit+ outputPrice:+ type: number+ format: doublenullable: true- description: The session identifier associated with the trace- release:- type: string+ description: Deprecated. See 'prices' instead. Price (USD) per output unit+ totalPrice:+ type: number+ format: doublenullable: true- description: The release version of the application when the trace was created- version:+ description: >-+ Deprecated. See 'prices' instead. Price (USD) per total unit. Cannot+ be set if input or output price is set.+ tokenizerId:type: stringnullable: true- description: The version of the trace- userId:+ description: >-+ Optional. Tokenizer to be applied to observations which match to+ this model. See docs for more details.+ tokenizerConfig:+ description: >-+ Optional. Configuration for the selected tokenizer. Needs to be+ JSON. See docs for more details.+ isLangfuseManaged:+ type: boolean+ createdAt:type: string- nullable: true- description: The user identifier associated with the trace- metadata:- nullable: true- description: The metadata associated with the trace. Can be any JSON.- tags:+ format: date-time+ description: Timestamp when the model was created+ prices:+ type: object+ additionalProperties:+ $ref: '#/components/schemas/ModelPrice'+ description: >-+ Deprecated. Use 'pricingTiers' instead for models with usage-based+ pricing variations.+++ This field shows prices by usage type from the default pricing tier.+ Maintained for backward compatibility.++ If the model uses tiered pricing, this field will be populated from+ the default tier's prices.+ pricingTiers:type: arrayitems:- type: string- description: The tags associated with the trace.- public:- type: boolean- description: Public traces are accessible via url without login- environment:- type: string+ $ref: '#/components/schemas/PricingTier'description: >-- The environment from which this trace originated. Can be any- lowercase alphanumeric string with hyphens and underscores that does- not start with 'langfuse'.+ Array of pricing tiers with conditional pricing based on usage+ thresholds or observation attributes.+++ Pricing tiers enable accurate cost tracking for models that charge+ different rates based on usage patterns++ or request attributes (e.g., high-volume usage, large context+ windows, cached tokens, or service tiers).+++ Each model must have exactly one default tier (isDefault=true,+ priority=0) that serves as a fallback.++ Additional conditional tiers can be defined with specific matching+ criteria.+++ If this array is empty, the model uses legacy flat pricing from the+ inputPrice/outputPrice/totalPrice fields.required:- id- - timestamp- - name- - sessionId- - release- - version- - userId- - tags- - public- - environment- TraceWithDetails:- title: TraceWithDetails+ - modelName+ - matchPattern+ - startDate+ - inputPrice+ - outputPrice+ - totalPrice+ - tokenizerId+ - tokenizerConfig+ - isLangfuseManaged+ - createdAt+ - prices+ - pricingTiers+ ModelPrice:+ title: ModelPrice+ type: object+ properties:+ price:+ type: number+ format: double+ required:+ - price+ PricingTierCondition:+ title: PricingTierCondition+ oneOf:+ - $ref: '#/components/schemas/PricingTierUsageCondition'+ - $ref: '#/components/schemas/PricingTierAttributeCondition'+ description: >-+ Condition for matching a pricing tier against usage details or+ observation attributes.+++ Usage-detail conditions treat usageDetailPattern as a regex, sum all+ matching usage values, and compare the sum to the numeric value.+ Model-parameter and metadata conditions match an exact top-level key+ against one or more string values.+ PricingTierUsageCondition:+ title: PricingTierUsageConditiontype: object+ description: Condition that sums usage details whose keys match a regex.properties:- htmlPath:+ usageDetailPattern:type: string- description: Path of trace in Langfuse UI- latency:- type: number- format: double- nullable: true- description: Latency of trace in seconds- totalCost:+ description: Regex pattern matched against usage detail keys.+ operator:+ $ref: '#/components/schemas/PricingTierOperator'+ value:type: numberformat: double- nullable: true- description: Cost of trace in USD- observations:- type: array- items:- type: string- nullable: true- description: List of observation ids- scores:- type: array- items:- type: string- nullable: true- description: List of score ids+ description: Numeric threshold for the summed matching usage values.+ caseSensitive:+ type: boolean+ description: Whether the usage-detail regex is case-sensitive.required:- - htmlPath- allOf:- - $ref: '#/components/schemas/Trace'- TraceWithFullDetails:- title: TraceWithFullDetails+ - usageDetailPattern+ - operator+ - value+ - caseSensitive+ PricingTierAttributeCondition:+ title: PricingTierAttributeConditiontype: object+ description: >-+ Condition that matches any configured value for a top-level observation+ attribute.properties:- htmlPath:+ source:+ $ref: '#/components/schemas/PricingTierAttributeSource'+ description: Observation attribute object evaluated by this condition.+ key:type: string- description: Path of trace in Langfuse UI- latency:- type: number- format: double- nullable: true- description: Latency of trace in seconds- totalCost:- type: number- format: double- nullable: true- description: Cost of trace in USD- observations:- type: array- items:- $ref: '#/components/schemas/ObservationsView'- description: List of observations- scores:+ description: Exact top-level attribute key.+ operator:+ type: string+ const: in+ description: Membership operator.+ values:type: arrayitems:- $ref: '#/components/schemas/ScoreV1'- description: List of scores- _deprecation:- $ref: '#/components/schemas/Deprecation'- nullable: true+ type: string+ description: Accepted string attribute values. At least one value is required.required:- - htmlPath- - observations- - scores- allOf:- - $ref: '#/components/schemas/Trace'- Session:- title: Session+ - source+ - key+ - operator+ - values+ PricingTierAttributeSource:+ title: PricingTierAttributeSource+ type: string+ enum:+ - model_parameters+ - metadata+ PricingTierConditionInput:+ title: PricingTierConditionInput+ oneOf:+ - $ref: '#/components/schemas/PricingTierUsageConditionInput'+ - $ref: '#/components/schemas/PricingTierAttributeCondition'+ description: Condition input for matching a pricing tier.+ PricingTierUsageConditionInput:+ title: PricingTierUsageConditionInputtype: object+ description: Input condition that sums usage details whose keys match a regex.properties:- id:- type: string- createdAt:- type: string- format: date-time- projectId:- type: string- environment:+ usageDetailPattern:type: string- description: The environment from which this session originated.- required:- - id- - createdAt- - projectId- - environment- SessionWithTraces:- title: SessionWithTraces- type: object- properties:- traces:- type: array- items:- $ref: '#/components/schemas/Trace'- _deprecation:- $ref: '#/components/schemas/Deprecation'+ description: Regex pattern matched against usage detail keys.+ operator:+ $ref: '#/components/schemas/PricingTierOperator'+ value:+ type: number+ format: double+ description: Numeric threshold for the summed matching usage values.+ caseSensitive:+ type: booleannullable: true+ description: Whether the usage-detail regex is case-sensitive. Defaults to false.required:- - traces- allOf:- - $ref: '#/components/schemas/Session'- Observation:- title: Observation+ - usageDetailPattern+ - operator+ - value+ PricingTier:+ title: PricingTiertype: object+ description: >-+ Pricing tier definition with conditional pricing based on usage+ thresholds or observation attributes.+++ Pricing tiers enable accurate cost tracking for LLM providers that+ charge different rates based on usage patterns or request attributes.++ For example, some providers charge higher rates when context size+ exceeds certain thresholds or when an accelerated service tier is+ selected.+++ How tier matching works:++ 1. Tiers are evaluated in ascending priority order (priority 1 before+ priority 2, etc.)++ 2. The first tier where ALL conditions match is selected++ 3. If no conditional tiers match, the default tier is used as a fallback++ 4. The default tier has priority 0 and no conditions+++ Why priorities matter:++ - Lower priority numbers are evaluated first, allowing you to define+ specific cases before general ones++ - Example: Priority 1 for "high usage" (>200K tokens), Priority 2 for+ "medium usage" (>100K tokens), Priority 0 for default++ - Without proper ordering, a less specific condition might match before+ a more specific one+++ Every model must have exactly one default tier to ensure cost+ calculation always succeeds.properties:id:type: string- description: The unique identifier of the observation- traceId:- type: string- nullable: true- description: The trace ID associated with the observation- type:- type: string- description: The type of the observation+ description: Unique identifier for the pricing tiername:type: string- nullable: true- description: The name of the observation- startTime:- type: string- format: date-time- description: The start time of the observation- endTime:- type: string- format: date-time- nullable: true- description: The end time of the observation.- completionStartTime:- type: string- format: date-time- nullable: true- description: The completion start time of the observation- model:- type: string- nullable: true- description: The model used for the observation- modelParameters:- description: The parameters of the model used for the observation- input:- description: The input data of the observation- version:- type: string- nullable: true- description: The version of the observation- metadata:- description: Additional metadata of the observation- output:- description: The output data of the observation- usage:- $ref: '#/components/schemas/Usage'description: >-- (Deprecated. Use usageDetails and costDetails instead.) The usage- data of the observation- level:- $ref: '#/components/schemas/ObservationLevel'- description: The level of the observation- statusMessage:- type: string- nullable: true- description: The status message of the observation- parentObservationId:- type: string- nullable: true- description: The parent observation ID- promptId:- type: string- nullable: true- description: The prompt ID associated with the observation- usageDetails:- type: object- additionalProperties:- type: integer+ Name of the pricing tier for display and identification purposes.+++ Examples: "Standard", "High Volume Tier", "Large Context", "Extended+ Context Tier"+ isDefault:+ type: boolean+ description: >-+ Whether this is the default tier. Every model must have exactly one+ default tier with priority 0 and no conditions.+++ The default tier serves as a fallback when no conditional tiers+ match, ensuring cost calculation always succeeds.++ It typically represents the base pricing for standard usage+ patterns.+ priority:+ type: integer+ description: >-+ Priority for tier matching evaluation. Lower numbers = higher+ priority (evaluated first).+++ The default tier must always have priority 0. Conditional tiers+ should have priority 1, 2, 3, etc.+++ Example ordering:++ - Priority 0: Default tier (no conditions, always matches as+ fallback)++ - Priority 1: High usage tier (e.g., >200K tokens)++ - Priority 2: Medium usage tier (e.g., >100K tokens)+++ This ensures more specific conditions are checked before general+ ones.+ conditions:+ type: array+ items:+ $ref: '#/components/schemas/PricingTierCondition'description: >-- The usage details of the observation. Key is the name of the usage- metric, value is the number of units consumed. The total key is the- sum of all (non-total) usage metrics or the total value ingested.- costDetails:+ Array of conditions that must ALL be met for this tier to match (AND+ logic).+++ The default tier must have an empty conditions array. Conditional+ tiers should have one or more conditions++ that define when this tier's pricing applies.+++ Multiple conditions enable complex matching scenarios (e.g., "high+ input tokens AND low output tokens").+ prices:type: objectadditionalProperties:type: numberformat: doubledescription: >-- The cost details of the observation. Key is the name of the cost- metric, value is the cost in USD. The total key is the sum of all- (non-total) cost metrics or the total value ingested.- environment:- type: string- description: >-- The environment from which this observation originated. Can be any- lowercase alphanumeric string with hyphens and underscores that does- not start with 'langfuse'.+ Prices (USD) by usage type for this tier.+++ Common usage types: "input", "output", "total", "request", "image"++ Prices are specified in USD per unit (e.g., per token, per request,+ per second).+++ Example: {"input": 0.000003, "output": 0.000015} means $3 per+ million input tokens and $15 per million output tokens.required:- id- - traceId- - type- name- - startTime- - endTime- - completionStartTime- - model- - modelParameters- - input- - version- - metadata- - output- - usage- - level- - statusMessage- - parentObservationId- - promptId- - usageDetails- - costDetails- - environment- ObservationsView:- title: ObservationsView+ - isDefault+ - priority+ - conditions+ - prices+ PricingTierInput:+ title: PricingTierInputtype: object+ description: >-+ Input schema for creating a pricing tier. The tier ID will be+ automatically generated server-side.+++ When creating a model with pricing tiers:++ - Exactly one tier must have isDefault=true (the fallback tier)++ - The default tier must have priority=0 and conditions=[]++ - All tier names and priorities must be unique within the model++ - Each tier must define at least one price+++ See PricingTier for detailed information about how tiers work and why+ they're useful.properties:- promptName:- type: string- nullable: true- description: The name of the prompt associated with the observation- promptVersion:- type: integer- nullable: true- description: The version of the prompt associated with the observation- modelId:+ name:type: string- nullable: true- description: The unique identifier of the model- inputPrice:- type: number- format: double- nullable: true- description: The price of the input in USD- outputPrice:- type: number- format: double- nullable: true- description: The price of the output in USD.- totalPrice:- type: number- format: double- nullable: true- description: The total price in USD.- calculatedInputCost:- type: number- format: double- nullable: truedescription: >-- (Deprecated. Use usageDetails and costDetails instead.) The- calculated cost of the input in USD- calculatedOutputCost:- type: number- format: double- nullable: true- description: >-- (Deprecated. Use usageDetails and costDetails instead.) The- calculated cost of the output in USD- calculatedTotalCost:- type: number- format: double+ Name of the pricing tier for display and identification purposes.+++ Must be unique within the model. Common patterns: "Standard", "High+ Volume Tier", "Extended Context"+ isDefault:+ type: booleannullable: truedescription: >-- (Deprecated. Use usageDetails and costDetails instead.) The- calculated total cost in USD- latency:- type: number- format: double- nullable: true- description: The latency in seconds.- timeToFirstToken:- type: number- format: double- nullable: true- description: The time to the first token in seconds- required:- - promptName- - promptVersion- - modelId- - inputPrice- - outputPrice- - totalPrice- - calculatedInputCost- - calculatedOutputCost- - calculatedTotalCost- - latency- - timeToFirstToken- allOf:- - $ref: '#/components/schemas/Observation'- ObservationsViewSingle:- title: ObservationsViewSingle- type: object- properties:- _deprecation:- $ref: '#/components/schemas/Deprecation'- nullable: true- allOf:- - $ref: '#/components/schemas/ObservationsView'- ObservationV2:- title: ObservationV2- type: object- description: >-- An observation from the v2 API with field-group-based selection.+ Whether this is the default tier. Exactly one tier per model must be+ marked as default.- Core fields are always present. Other fields are included only when- their field group is requested.- properties:- id:- type: string- description: The unique identifier of the observation- traceId:- type: string- nullable: true- description: The trace ID associated with the observation- startTime:- type: string- format: date-time- description: The start time of the observation- endTime:- type: string- format: date-time- nullable: true- description: The end time of the observation- projectId:- type: string- description: The project ID this observation belongs to- parentObservationId:- type: string- nullable: true++ Requirements for default tier:++ - Must have isDefault=true++ - Must have priority=0++ - Must have empty conditions array (conditions=[])+++ The default tier acts as a fallback when no conditional tiers match.+ Defaults to false when omitted.+ priority:+ type: integerdescription: >-- The physical parent observation ID, if present.+ Priority for tier matching evaluation. Lower numbers = higher+ priority (evaluated first).- Observations marked as app roots by the SDK may retain a non-null- parent ID.- type:- type: string- description: The type of the observation (e.g. GENERATION, SPAN, EVENT)- isRootObservation:- type: boolean- nullable: true++ Must be unique within the model. The default tier must have+ priority=0.++ Conditional tiers should use priority 1, 2, 3, etc. based on their+ specificity.+ conditions:+ type: array+ items:+ $ref: '#/components/schemas/PricingTierConditionInput'description: >-- Whether this observation is a logical root.+ Array of conditions that must ALL be met for this tier to match (AND+ logic).- This is true for observations without a physical parent and- observations marked as app roots by the SDK.- name:- type: string- nullable: true- description: The name of the observation- level:- $ref: '#/components/schemas/ObservationLevel'- nullable: true- description: The level of the observation- statusMessage:- type: string- nullable: true- description: The status message of the observation- version:- type: string- nullable: true- description: The version of the observation- environment:- type: string- nullable: true- description: The environment from which this observation originated- bookmarked:- type: boolean++ The default tier must have an empty array (conditions=[]).++ Conditional tiers should define one or more conditions that specify+ when this tier's pricing applies.+++ Conditions can compare summed matching usage details to a numeric+ threshold, or exactly match a top-level model parameter or metadata+ value.+ prices:+ type: object+ additionalProperties:+ type: number+ format: double+ description: >-+ Prices (USD) by usage type for this tier. At least one price must be+ defined.+++ Common usage types: "input", "output", "total", "request", "image"++ Prices are in USD per unit (e.g., per token).+++ Example: {"input": 0.000003, "output": 0.000015} represents $3 per+ million input tokens and $15 per million output tokens.+ required:+ - name+ - priority+ - conditions+ - prices+ PricingTierOperator:+ title: PricingTierOperator+ type: string+ enum:+ - gt+ - gte+ - lt+ - lte+ - eq+ - neq+ description: Comparison operators for pricing tier conditions+ ModelUsageUnit:+ title: ModelUsageUnit+ type: string+ enum:+ - CHARACTERS+ - TOKENS+ - MILLISECONDS+ - SECONDS+ - IMAGES+ - REQUESTS+ description: Unit of usage in Langfuse+ ObservationLevel:+ title: ObservationLevel+ type: string+ enum:+ - DEBUG+ - DEFAULT+ - WARNING+ - ERROR+ MapValue:+ title: MapValue+ oneOf:+ - type: stringnullable: true- description: Whether the observation is bookmarked- public:- type: boolean+ - type: integernullable: true- description: Whether the observation is public- userId:- type: string+ - type: number+ format: floatnullable: true- description: The user ID associated with the observation- sessionId:- type: string+ - type: booleannullable: true- description: The session ID associated with the observation- completionStartTime:- type: string- format: date-time+ - type: array+ items:+ type: stringnullable: true- description: The completion start time of the observation- createdAt:+ CommentObjectType:+ title: CommentObjectType+ type: string+ enum:+ - TRACE+ - OBSERVATION+ - SESSION+ - PROMPT+ DatasetStatus:+ title: DatasetStatus+ type: string+ enum:+ - ACTIVE+ - ARCHIVED+ ScoreSource:+ title: ScoreSource+ type: string+ enum:+ - ANNOTATION+ - API+ - EVAL+ ScoreConfigDataType:+ title: ScoreConfigDataType+ type: string+ enum:+ - NUMERIC+ - BOOLEAN+ - CATEGORICAL+ - TEXT+ ScoreDataType:+ title: ScoreDataType+ type: string+ enum:+ - NUMERIC+ - BOOLEAN+ - CATEGORICAL+ - CORRECTION+ - TEXT+ DeleteDatasetItemResponse:+ title: DeleteDatasetItemResponse+ type: object+ properties:+ message:type: string- format: date-time- nullable: true- description: The creation timestamp of the observation- updatedAt:+ description: Success message after deletion+ required:+ - message+ CreateDatasetItemRequest:+ title: CreateDatasetItemRequest+ type: object+ properties:+ datasetName:type: string- format: date-time- nullable: true- description: The last update timestamp of the observationinput:nullable: true- description: The input data of the observation- output:+ expectedOutput:nullable: true- description: The output data of the observationmetadata:nullable: true- description: Additional metadata of the observation- providedModelName:+ sourceTraceId:type: stringnullable: true- description: The model name as provided by the user- internalModelId:+ sourceObservationId:type: stringnullable: true- description: The internal model ID matched by Langfuse- modelParameters:- nullable: true- description: The parameters of the model used for the observation- usageDetails:- type: object- additionalProperties:- type: integer- nullable: true- description: >-- The usage details of the observation. Key is the usage metric name,- value is the number of units consumed.- costDetails:- type: object- additionalProperties:- type: number- format: double- nullable: true- description: >-- The cost details of the observation. Key is the cost metric name,- value is the cost in USD.- totalCost:- type: number- format: double- nullable: true- description: The total cost of the observation in USD- usagePricingTierName:+ id:type: stringnullable: truedescription: >-- The name of the pricing tier applied to this observation's usage- costs- promptId:+ Dataset items are upserted on their id. Id needs to be unique+ (project-level), cannot be reused across datasets, and must be at+ most 255 characters.+ status:+ $ref: '#/components/schemas/DatasetStatus'+ nullable: true+ description: Defaults to ACTIVE for newly created items+ required:+ - datasetName+ PaginatedDatasetItems:+ title: PaginatedDatasetItems+ type: object+ properties:+ data:+ type: array+ items:+ $ref: '#/components/schemas/DatasetItem'+ meta:+ $ref: '#/components/schemas/utilsMetaResponse'+ required:+ - data+ - meta+ CreateDatasetRunItemRequest:+ title: CreateDatasetRunItemRequest+ type: object+ properties:+ runName:type: string- nullable: true- description: The prompt ID associated with the observation- promptName:+ runDescription:type: stringnullable: true- description: The prompt name associated with the observation- promptVersion:- type: integer- nullable: true- description: The prompt version associated with the observation- latency:- type: number- format: double+ description: Description of the run. If run exists, description will be updated.+ metadata:nullable: true- description: The latency in seconds- timeToFirstToken:- type: number- format: double+ description: Metadata of the dataset run, updates run if run already exists+ datasetItemId:+ type: string+ observationId:+ type: stringnullable: true- description: The time to first token in seconds- modelId:+ traceId:type: stringnullable: truedescription: >-- The matched model ID. Null when the `model` field group is not- requested.- inputPrice:+ traceId should always be provided. For compatibility with older SDK+ versions it can also be inferred from the provided observationId.+ datasetVersion:type: string+ format: date-timenullable: truedescription: >-- The input token price (USD per unit) from the matched model,- serialized as a decimal string (e.g. "0.0001"). Null when the- `model` field group is not requested.- outputPrice:+ ISO 8601 timestamp (RFC 3339, Section 5.6) in UTC (e.g.,+ "2026-01-21T14:35:42Z").++ Specifies the dataset version to use for this experiment run.++ If provided, the experiment will use dataset items as they existed+ at or before this timestamp.++ If not provided, uses the latest version of dataset items.+ createdAt:type: string+ format: date-timenullable: truedescription: >-- The output token price (USD per unit) from the matched model,- serialized as a decimal string (e.g. "0.0001"). Null when the- `model` field group is not requested.- totalPrice:+ Optional timestamp to set the createdAt field of the dataset run+ item. If not provided or null, defaults to current timestamp.+ required:+ - runName+ - datasetItemId+ PaginatedDatasetRunItems:+ title: PaginatedDatasetRunItems+ type: object+ properties:+ data:+ type: array+ items:+ $ref: '#/components/schemas/DatasetRunItem'+ meta:+ $ref: '#/components/schemas/utilsMetaResponse'+ _deprecation:+ $ref: '#/components/schemas/Deprecation'+ nullable: true+ required:+ - data+ - meta+ PaginatedDatasets:+ title: PaginatedDatasets+ type: object+ properties:+ data:+ type: array+ items:+ $ref: '#/components/schemas/Dataset'+ meta:+ $ref: '#/components/schemas/utilsMetaResponse'+ required:+ - data+ - meta+ CreateDatasetRequest:+ title: CreateDatasetRequest+ type: object+ properties:+ name:+ type: string+ description:type: stringnullable: true+ metadata:+ nullable: true+ inputSchema:+ nullable: truedescription: >-- The total token price (USD per unit) from the matched model,- serialized as a decimal string (e.g. "0.0001"). Null when the- `model` field group is not requested.- traceName:- type: string+ JSON Schema for validating dataset item inputs. When set, all new+ and existing dataset items will be validated against this schema.+ expectedOutputSchema:nullable: true- description: The name of the parent trace- tags:+ description: >-+ JSON Schema for validating dataset item expected outputs. When set,+ all new and existing dataset items will be validated against this+ schema.+ required:+ - name+ PaginatedDatasetRuns:+ title: PaginatedDatasetRuns+ type: object+ properties:+ data:type: arrayitems:- type: string- nullable: true- description: Tags from the parent trace (denormalized onto the observation)- release:- type: string+ $ref: '#/components/schemas/DatasetRun'+ meta:+ $ref: '#/components/schemas/utilsMetaResponse'+ _deprecation:+ $ref: '#/components/schemas/Deprecation'nullable: true- description: The release version of the parent tracerequired:- - id- - traceId- - startTime- - endTime- - projectId- - parentObservationId- - type- - modelId- - inputPrice- - outputPrice- - totalPrice- Usage:- title: Usage+ - data+ - meta+ DeleteDatasetRunResponse:+ title: DeleteDatasetRunResponsetype: object+ properties:+ message:+ type: string+ required:+ - message+ EvaluatorType:+ title: EvaluatorType+ type: string+ enum:+ - llm_as_judge+ - code+ description: |-+ The evaluator type.++ The public API supports LLM-as-a-judge and code evaluators.+ CodeEvaluatorSourceCodeLanguage:+ title: CodeEvaluatorSourceCodeLanguage+ type: string+ enum:+ - PYTHON+ - TYPESCRIPT+ description: Code evaluator runtime language.+ PromptVariableMappingSource:+ title: PromptVariableMappingSource+ type: string+ enum:+ - input+ - output+ - metadata+ - tool_calls+ - expected_output+ - experiment_item_metadatadescription: >-- (Deprecated. Use usageDetails and costDetails instead.) Standard- interface for usage and cost+ Source field used to populate a prompt variable.+++ Use these values when mapping evaluator prompt variables to live data.+++ Source semantics:++ - `input`: the observation input payload++ - `output`: the observation output payload++ - `metadata`: the observation metadata object. Combine with `jsonPath`+ when you need one nested field instead of the whole object.++ - `tool_calls`: the tool calls recorded on the observation, as an array+ of `{id, name, arguments, type, index}` objects in the order the model+ emitted them. Combine with `jsonPath` (for example `$[*].name`) to+ select parts of each call.++ - `expected_output`: the experiment item's expected output when the+ observation belongs to an experiment.++ - `experiment_item_metadata`: the experiment item's metadata when the+ observation belongs to an experiment.+ EvaluatorOutputScoreType:+ title: EvaluatorOutputScoreType+ type: string+ enum:+ - NUMERIC+ - BOOLEAN+ - CATEGORICAL+ description: >-+ Structured score type returned by an evaluator.+++ This controls the type of score value Langfuse stores for evaluation+ results:++ - `NUMERIC`: a numeric score such as `0.82`++ - `BOOLEAN`: a boolean score such as `true`++ - `CATEGORICAL`: one or more category labels from a fixed list+ EvaluatorOutputDefinition:+ title: EvaluatorOutputDefinition+ oneOf:+ - type: object+ allOf:+ - type: object+ properties:+ dataType:+ type: string+ enum:+ - NUMERIC+ - $ref: '#/components/schemas/PublicEvaluatorNumericScore'+ required:+ - dataType+ - type: object+ allOf:+ - type: object+ properties:+ dataType:+ type: string+ enum:+ - BOOLEAN+ - $ref: '#/components/schemas/PublicEvaluatorBooleanScore'+ required:+ - dataType+ - type: object+ allOf:+ - type: object+ properties:+ dataType:+ type: string+ enum:+ - CATEGORICAL+ - $ref: '#/components/schemas/PublicEvaluatorCategoricalScore'+ required:+ - dataType+ description: >-+ Flat structured output definition used when creating or updating an+ evaluator.+++ - `dataType` is required.++ - `scoreReasoningInstructions` and `scoreValueInstructions` are optional+ instructions.++ - `minValue` and `maxValue` apply only to `NUMERIC` outputs. If both are+ set, `minValue` must not exceed `maxValue`.++ - `categories` and `shouldAllowMultipleMatches` apply only to+ `CATEGORICAL` outputs.++ - Do not send `version`; that is an internal storage detail.+ EvaluatorOutputDefinitionBase:+ title: EvaluatorOutputDefinitionBase+ type: objectproperties:- input:- type: integer- description: Number of input units (e.g. tokens)- output:- type: integer- description: Number of output units (e.g. tokens)- total:- type: integer- description: Defaults to input+output if not set- unit:+ scoreReasoningInstructions:type: stringnullable: true- description: Unit of measurement- inputCost:- type: number- format: double- nullable: true- description: USD input cost- outputCost:- type: number- format: double- nullable: true- description: USD output cost- totalCost:- type: number- format: double+ description: >-+ Optional instructions for deriving the reasoning returned with the+ score.+ scoreValueInstructions:+ type: stringnullable: true- description: USD total cost, defaults to input+output- required:- - input- - output- - total- - unit- ScoreConfig:- title: ScoreConfig+ description: Optional instructions for deriving the score value.+ PublicEvaluatorNumericScore:+ title: PublicEvaluatorNumericScoretype: object- description: Configuration for a scoreproperties:- id:- type: string- name:- type: string- createdAt:- type: string- format: date-time- updatedAt:- type: string- format: date-time- projectId:- type: stringdataType:- $ref: '#/components/schemas/ScoreConfigDataType'- isArchived:- type: boolean- description: Whether the score config is archived. Defaults to false+ type: string+ const: NUMERIC+ description: Numeric score output.minValue:type: numberformat: doublenullable: true- description: >-- Sets minimum value for numerical scores. If not set, the minimum- value defaults to -∞+ description: Optional inclusive minimum value.maxValue:type: numberformat: doublenullable: true- description: >-- Sets maximum value for numerical scores. If not set, the maximum- value defaults to +∞- categories:- type: array- items:- $ref: '#/components/schemas/ConfigCategory'- nullable: true- description: Configures custom categories for categorical scores- description:- type: string- nullable: true- description: Description of the score config+ description: Optional inclusive maximum value.required:- - id- - name- - createdAt- - updatedAt- - projectId- dataType- - isArchived- ConfigCategory:- title: ConfigCategory- type: object- properties:- value:- type: number- format: double- label:- type: string- required:- - value- - label- BaseScoreV1:- title: BaseScoreV1- type: object- properties:- id:- type: string- traceId:- type: string- name:- type: string- source:- $ref: '#/components/schemas/ScoreSource'- observationId:- type: string- nullable: true- description: The observation ID associated with the score- timestamp:- type: string- format: date-time- createdAt:- type: string- format: date-time- updatedAt:- type: string- format: date-time- authorUserId:- type: string- nullable: true- description: The user ID of the author- comment:- type: string- nullable: true- description: Comment on the score- metadata:- description: Metadata associated with the score- configId:- type: string- nullable: true- description: >-- Reference a score config on a score. When set, config and score name- must be equal and value must comply to optionally defined numerical- range- queueId:- type: string- nullable: true- description: >-- The annotation queue referenced by the score. Indicates if score was- initially created while processing annotation queue.- environment:- type: string- description: >-- The environment from which this score originated. Can be any- lowercase alphanumeric string with hyphens and underscores that does- not start with 'langfuse'.- required:- - id- - traceId- - name- - source- - timestamp- - createdAt- - updatedAt- - authorUserId- - comment- - metadata- - configId- - queueId- - environment- NumericScoreV1:- title: NumericScoreV1- type: object- properties:- value:- type: number- format: double- description: The numeric value of the score- required:- - value- allOf:- - $ref: '#/components/schemas/BaseScoreV1'- BooleanScoreV1:- title: BooleanScoreV1- type: object- properties:- value:- type: number- format: double- description: >-- The numeric value of the score. Equals 1 for "True" and 0 for "False"- stringValue:- type: string- description: >-- The string representation of the score value. Is inferred from the- numeric value and equals "True" or "False"- required:- - value- - stringValueallOf:- - $ref: '#/components/schemas/BaseScoreV1'- CategoricalScoreV1:- title: CategoricalScoreV1+ - $ref: '#/components/schemas/EvaluatorOutputDefinitionBase'+ PublicEvaluatorBooleanScore:+ title: PublicEvaluatorBooleanScoretype: objectproperties:- value:- type: number- format: double- description: >-- Represents the numeric category mapping of the stringValue. If no- config is linked, defaults to 0.- stringValue:+ dataType:type: string- description: >-- The string representation of the score value. If no config is- linked, can be any string. Otherwise, must map to a config category+ const: BOOLEAN+ description: Boolean score output.required:- - value- - stringValue+ - dataTypeallOf:- - $ref: '#/components/schemas/BaseScoreV1'- TextScoreV1:- title: TextScoreV1+ - $ref: '#/components/schemas/EvaluatorOutputDefinitionBase'+ PublicEvaluatorCategoricalScore:+ title: PublicEvaluatorCategoricalScoretype: objectproperties:- stringValue:+ dataType:type: string- description: The text content of the score (1-500 characters)+ const: CATEGORICAL+ description: Categorical score output.+ categories:+ type: array+ items:+ type: string+ description: Allowed category values. At least two unique values are required.+ shouldAllowMultipleMatches:+ type: boolean+ description: Whether the evaluator may return more than one category.required:- - stringValue+ - dataType+ - categories+ - shouldAllowMultipleMatchesallOf:- - $ref: '#/components/schemas/BaseScoreV1'- ScoreV1:- title: ScoreV1+ - $ref: '#/components/schemas/EvaluatorOutputDefinitionBase'+ PublicEvaluatorOutputDefinition:+ title: PublicEvaluatorOutputDefinitiononeOf:- type: objectallOf:@@ -10084,18 +12448,7 @@ components:type: stringenum:- NUMERIC- - $ref: '#/components/schemas/NumericScoreV1'- required:- - dataType- - type: object- allOf:- - type: object- properties:- dataType:- type: string- enum:- - CATEGORICAL- - $ref: '#/components/schemas/CategoricalScoreV1'+ - $ref: '#/components/schemas/PublicEvaluatorNumericScore'required:- dataType- type: object@@ -10106,7 +12459,7 @@ components:type: stringenum:- BOOLEAN- - $ref: '#/components/schemas/BooleanScoreV1'+ - $ref: '#/components/schemas/PublicEvaluatorBooleanScore'required:- dataType- type: object@@ -10116,1236 +12469,1588 @@ components:dataType:type: stringenum:- - TEXT- - $ref: '#/components/schemas/TextScoreV1'+ - CATEGORICAL+ - $ref: '#/components/schemas/PublicEvaluatorCategoricalScore'required:- dataType- BaseScore:- title: BaseScore+ description: >-+ Flat evaluator output definition returned by the public API.+++ This response always includes `dataType` and never includes an internal+ output-definition `version`.++ Optional empty descriptions from legacy definitions are omitted.+ EvaluationRuleStringFilterOperator:+ title: EvaluationRuleStringFilterOperator+ type: string+ enum:+ - '='+ - contains+ - does not contain+ - starts with+ - ends with+ EvaluationRuleNumberFilterOperator:+ title: EvaluationRuleNumberFilterOperator+ type: string+ enum:+ - '='+ - '>'+ - <+ - '>='+ - <=+ EvaluationRuleOptionsFilterOperator:+ title: EvaluationRuleOptionsFilterOperator+ type: string+ enum:+ - any of+ - none of+ EvaluationRuleArrayOptionsFilterOperator:+ title: EvaluationRuleArrayOptionsFilterOperator+ type: string+ enum:+ - any of+ - none of+ - all of+ EvaluationRuleBooleanFilterOperator:+ title: EvaluationRuleBooleanFilterOperator+ type: string+ enum:+ - '='+ - <>+ EvaluationRuleNullFilterOperator:+ title: EvaluationRuleNullFilterOperator+ type: string+ enum:+ - is null+ - is not null+ DateTimeEvaluationRuleFilter:+ title: DateTimeEvaluationRuleFiltertype: objectproperties:- id:+ column:type: string- traceId:+ description: Column to filter on.+ operator:+ $ref: '#/components/schemas/EvaluationRuleNumberFilterOperator'+ description: Comparison operator for datetime values.+ value:type: string- nullable: true- description: The trace ID associated with the score- sessionId:+ format: date-time+ description: Datetime value to compare against.+ required:+ - column+ - operator+ - value+ StringEvaluationRuleFilter:+ title: StringEvaluationRuleFilter+ type: object+ properties:+ column:type: string- nullable: true- description: The session ID associated with the score- observationId:+ description: Column to filter on.+ operator:+ $ref: '#/components/schemas/EvaluationRuleStringFilterOperator'+ value:type: string- nullable: true- description: The observation ID associated with the score- datasetRunId:+ required:+ - column+ - operator+ - value+ NumberEvaluationRuleFilter:+ title: NumberEvaluationRuleFilter+ type: object+ properties:+ column:type: string- nullable: true- description: The dataset run ID associated with the score- name:+ description: Column to filter on.+ operator:+ $ref: '#/components/schemas/EvaluationRuleNumberFilterOperator'+ value:+ type: number+ format: double+ required:+ - column+ - operator+ - value+ StringOptionsEvaluationRuleFilter:+ title: StringOptionsEvaluationRuleFilter+ type: object+ properties:+ column:type: string- source:- $ref: '#/components/schemas/ScoreSource'- timestamp:+ description: Column to filter on.+ operator:+ $ref: '#/components/schemas/EvaluationRuleOptionsFilterOperator'+ value:+ type: array+ items:+ type: string+ description: One or more allowed string values.+ required:+ - column+ - operator+ - value+ ArrayOptionsEvaluationRuleFilter:+ title: ArrayOptionsEvaluationRuleFilter+ type: object+ properties:+ column:type: string- format: date-time- createdAt:+ description: Column to filter on.+ operator:+ $ref: '#/components/schemas/EvaluationRuleArrayOptionsFilterOperator'+ value:+ type: array+ items:+ type: string+ description: One or more array elements to match.+ required:+ - column+ - operator+ - value+ StringObjectEvaluationRuleFilter:+ title: StringObjectEvaluationRuleFilter+ type: object+ properties:+ column:type: string- format: date-time- updatedAt:+ description: >-+ Object-valued column to filter on. Currently only `metadata` is+ supported.+ key:type: string- format: date-time- authorUserId:+ description: Top-level key inside the object-valued column to filter on.+ operator:+ $ref: '#/components/schemas/EvaluationRuleStringFilterOperator'+ value:type: string- nullable: true- description: The user ID of the author- comment:+ required:+ - column+ - key+ - operator+ - value+ NumberObjectEvaluationRuleFilter:+ title: NumberObjectEvaluationRuleFilter+ type: object+ properties:+ column:type: string- nullable: true- description: Comment on the score- metadata:- description: Metadata associated with the score- configId:+ description: Object-valued column to filter on.+ key:type: string- nullable: true- description: >-- Reference a score config on a score. When set, config and score name- must be equal and value must comply to optionally defined numerical- range- queueId:+ description: Key inside the object-valued column to filter on.+ operator:+ $ref: '#/components/schemas/EvaluationRuleNumberFilterOperator'+ value:+ type: number+ format: double+ required:+ - column+ - key+ - operator+ - value+ CategoryOptionsEvaluationRuleFilter:+ title: CategoryOptionsEvaluationRuleFilter+ type: object+ properties:+ column:type: string- nullable: true- description: >-- The annotation queue referenced by the score. Indicates if score was- initially created while processing annotation queue.- environment:+ description: Object-valued column to filter on.+ key:type: string- description: >-- The environment from which this score originated. Can be any- lowercase alphanumeric string with hyphens and underscores that does- not start with 'langfuse'.+ description: Key inside the object-valued column to filter on.+ operator:+ $ref: '#/components/schemas/EvaluationRuleOptionsFilterOperator'+ value:+ type: array+ items:+ type: stringrequired:- - id- - name- - source- - timestamp- - createdAt- - updatedAt- - authorUserId- - comment- - metadata- - configId- - queueId- - environment- NumericScore:- title: NumericScore+ - column+ - key+ - operator+ - value+ BooleanEvaluationRuleFilter:+ title: BooleanEvaluationRuleFiltertype: objectproperties:+ column:+ type: string+ description: Column to filter on.+ operator:+ $ref: '#/components/schemas/EvaluationRuleBooleanFilterOperator'value:- type: number- format: double- description: The numeric value of the score+ type: booleanrequired:+ - column+ - operator- value- allOf:- - $ref: '#/components/schemas/BaseScore'- BooleanScore:- title: BooleanScore+ NullEvaluationRuleFilter:+ title: NullEvaluationRuleFiltertype: objectproperties:+ column:+ type: string+ description: Column to filter on, for example `parentObservationId`.+ operator:+ $ref: '#/components/schemas/EvaluationRuleNullFilterOperator'value:- type: number- format: double- description: >-- The numeric value of the score. Equals 1 for "True" and 0 for "False"- stringValue:type: string+ const: ''description: >-- The string representation of the score value. Is inferred from the- numeric value and equals "True" or "False"+ Required empty-string placeholder used by the runtime filter+ contract.required:+ - column+ - operator- value- - stringValue- allOf:- - $ref: '#/components/schemas/BaseScore'- CategoricalScore:- title: CategoricalScore+ PromptVariableMappingInput:+ title: PromptVariableMappingInputtype: object+ description: >-+ Connects one prompt variable to data from an observation or experiment.+++ Manual mappings are used for `llm_as_judge` evaluators. `code`+ evaluators use a fixed runtime mapping managed by Langfuse.+++ How to build a valid mapping list:++ 1. Create the evaluator or fetch it with `GET /evaluators/{id}`.++ 2. Read the evaluator `variables` array.++ 3. Add exactly one mapping object for each variable in that array.++ 4. Use the variable name exactly as returned, without braces such as+ `{{` or `}}`.++ 5. Choose the source field that should populate the variable.+++ `jsonPath` is optional. Use it only when the selected source is a JSON+ object and you want to extract one nested field before inserting it into+ the evaluator prompt.+++ Invalid, missing, or duplicate mappings return a validation error.+ Malformed JSONPath expressions are also rejected.properties:- value:- type: number- format: double+ variable:+ type: stringdescription: >-- Represents the numeric category mapping of the stringValue. If no- config is linked, defaults to 0.- stringValue:+ Prompt variable name without braces.+++ Example: for the prompt `Judge {{input}} against {{output}}`, use+ `input` and `output`.+ source:+ $ref: '#/components/schemas/PromptVariableMappingSource'+ description: >-+ Source field that should populate the prompt variable.+++ Available sources are `input`, `output`, `metadata`, `tool_calls`,+ `expected_output`, and `experiment_item_metadata`.+ jsonPath:type: string+ nullable: truedescription: >-- The string representation of the score value. If no config is- linked, can be any string. Otherwise, must map to a config category+ Optional JSONPath selector applied to the selected source before it+ is passed to the evaluator prompt.+++ Requirements:++ - Must start with `$`++ - Must be a syntactically valid JSONPath expression++ - Most useful with `source=metadata`required:- - value- - stringValue- allOf:- - $ref: '#/components/schemas/BaseScore'- CorrectionScore:- title: CorrectionScore+ - variable+ - source+ PromptVariableMappingRead:+ title: PromptVariableMappingReadtype: object+ description: |-+ Connects one prompt variable to source data.++ `source` is `null` when the mapping is incomplete.properties:- value:- type: number- format: double- description: The numeric value of the score. Always 0 for correction scores.- stringValue:+ variable:type: string- description: The string representation of the correction content+ description: Prompt variable name without braces.+ source:+ type: string+ nullable: true+ description: >-+ Stored source field populating the variable, or `null` when the+ mapping is incomplete.+ jsonPath:+ type: string+ nullable: true+ description: Optional JSONPath selector applied to the selected source.required:- - value- - stringValue- allOf:- - $ref: '#/components/schemas/BaseScore'- TextScore:- title: TextScore+ - variable+ - source+ PromptVariableMapping:+ title: PromptVariableMapping+ oneOf:+ - $ref: '#/components/schemas/PromptVariableMappingRead'+ - $ref: '#/components/schemas/LegacyPromptVariableMapping'+ description: Connects one prompt variable to source data.+ LegacyPromptVariableMapping:+ title: LegacyPromptVariableMappingtype: object+ description: >-+ **Deprecated:** Connects one prompt variable to data from a legacy trace+ or dataset evaluation rule.+++ `langfuseObject` selects the object kind. `objectName` separately+ selects a named observation and is `null` for trace and dataset-item+ mappings.properties:- stringValue:+ mappingType:type: string- description: The text content of the score (1-500 characters)+ const: legacy+ description: Explicitly marks this as a legacy mapping.+ variable:+ type: string+ description: Prompt variable name without braces.+ langfuseObject:+ $ref: '#/components/schemas/LegacyEvaluationObject'+ description: Legacy object kind selected as the mapping source.+ objectName:+ type: string+ nullable: true+ description: >-+ Observation name to match, or `null` when `langfuseObject` is+ `trace` or `dataset_item`.+ source:+ type: string+ description: Field selected from the legacy object.+ jsonPath:+ type: string+ nullable: true+ description: Optional JSONPath selector applied to the selected field.required:- - stringValue- allOf:- - $ref: '#/components/schemas/BaseScore'- Score:- title: Score+ - mappingType+ - variable+ - langfuseObject+ - objectName+ - source+ LegacyEvaluationObject:+ title: LegacyEvaluationObject+ type: string+ enum:+ - trace+ - span+ - generation+ - event+ - agent+ - tool+ - chain+ - retriever+ - evaluator+ - embedding+ - guardrail+ - dataset_item+ description: >-+ **Deprecated:** Legacy Langfuse object kind used by trace and dataset+ evaluation rules.+ EvaluationRuleFilter:+ title: EvaluationRuleFilteroneOf:- type: objectallOf:- type: objectproperties:- dataType:+ type:type: stringenum:- - NUMERIC- - $ref: '#/components/schemas/NumericScore'+ - datetime+ - $ref: '#/components/schemas/DateTimeEvaluationRuleFilter'required:- - dataType+ - type- type: objectallOf:- type: objectproperties:- dataType:+ type:type: stringenum:- - CATEGORICAL- - $ref: '#/components/schemas/CategoricalScore'+ - string+ - $ref: '#/components/schemas/StringEvaluationRuleFilter'required:- - dataType+ - type- type: objectallOf:- type: objectproperties:- dataType:+ type:type: stringenum:- - BOOLEAN- - $ref: '#/components/schemas/BooleanScore'+ - number+ - $ref: '#/components/schemas/NumberEvaluationRuleFilter'required:- - dataType+ - type- type: objectallOf:- type: objectproperties:- dataType:+ type:type: stringenum:- - CORRECTION- - $ref: '#/components/schemas/CorrectionScore'+ - stringOptions+ - $ref: '#/components/schemas/StringOptionsEvaluationRuleFilter'required:- - dataType+ - type- type: objectallOf:- type: objectproperties:- dataType:+ type:type: stringenum:- - TEXT- - $ref: '#/components/schemas/TextScore'+ - categoryOptions+ - $ref: '#/components/schemas/CategoryOptionsEvaluationRuleFilter'required:- - dataType- properties:- _deprecation:- $ref: '#/components/schemas/Deprecation'- nullable: true- CreateScoreValue:- title: CreateScoreValue- oneOf:- - type: number- format: double- - type: string- description: >-- The value of the score. Must be passed as string for categorical and- text scores, and numeric for boolean and numeric scores- Comment:- title: Comment- type: object- properties:- id:- type: string- projectId:- type: string- createdAt:- type: string- format: date-time- updatedAt:- type: string- format: date-time- objectType:- $ref: '#/components/schemas/CommentObjectType'- objectId:- type: string- content:- type: string- authorUserId:- type: string- nullable: true- description: The user ID of the comment author- required:- - id- - projectId- - createdAt- - updatedAt- - objectType- - objectId- - content- Dataset:- title: Dataset- type: object- properties:- id:- type: string- name:- type: string- description:- type: string- nullable: true- description: Description of the dataset- metadata:- description: Metadata associated with the dataset- inputSchema:- nullable: true- description: JSON Schema for validating dataset item inputs- expectedOutputSchema:- nullable: true- description: JSON Schema for validating dataset item expected outputs- projectId:- type: string- createdAt:- type: string- format: date-time- updatedAt:- type: string- format: date-time- required:- - id- - name- - description- - metadata- - inputSchema- - expectedOutputSchema- - projectId- - createdAt- - updatedAt- DatasetItem:- title: DatasetItem+ - type+ - type: object+ allOf:+ - type: object+ properties:+ type:+ type: string+ enum:+ - arrayOptions+ - $ref: '#/components/schemas/ArrayOptionsEvaluationRuleFilter'+ required:+ - type+ - type: object+ allOf:+ - type: object+ properties:+ type:+ type: string+ enum:+ - stringObject+ - $ref: '#/components/schemas/StringObjectEvaluationRuleFilter'+ required:+ - type+ - type: object+ allOf:+ - type: object+ properties:+ type:+ type: string+ enum:+ - numberObject+ - $ref: '#/components/schemas/NumberObjectEvaluationRuleFilter'+ required:+ - type+ - type: object+ allOf:+ - type: object+ properties:+ type:+ type: string+ enum:+ - boolean+ - $ref: '#/components/schemas/BooleanEvaluationRuleFilter'+ required:+ - type+ - type: object+ allOf:+ - type: object+ properties:+ type:+ type: string+ enum:+ - 'null'+ - $ref: '#/components/schemas/NullEvaluationRuleFilter'+ required:+ - type+ description: >-+ One filter condition used to decide whether a live-ingested observation+ should be evaluated.+++ All filters must be satisfied for the rule to run. Experiment scope is+ expressed with filters rather than a separate target field:++ - `isExperimentItemRootSpan = true` limits execution to experiment item+ roots.++ - `datasetId` limits execution to experiments for the selected datasets.+ Use dataset IDs from `GET /api/public/v2/datasets`.+++ Pick the filter `type` first, as it determines the required fields and+ value shape. Use `key` only for object filters such as `metadata`.+++ Operator quick reference by filter `type`:++ - `string`: `"="`, `contains`, `does not contain`, `starts with`, `ends+ with`++ - `number`: `"="`, `">"`, `"<"`, `">="`, `"<="`++ - `datetime`: `"="`, `">"`, `"<"`, `">="`, `"<="`++ - `stringOptions`: `any of`, `none of`++ - `arrayOptions`: `any of`, `none of`, `all of`++ - `stringObject`: same operators as `string`++ - `boolean`: `"="`, `"<>"`++ - `null`: `is null`, `is not null`+++ Supported columns. Each column accepts exactly one filter `type`;+ sending a column with a different `type` is rejected with `400`.++ - `type`: `stringOptions`, values `SPAN`, `EVENT`, `GENERATION`,+ `AGENT`, `TOOL`, `CHAIN`, `RETRIEVER`, `EVALUATOR`, `EMBEDDING`,+ `GUARDRAIL`++ - `name`: `stringOptions`++ - `environment`: `stringOptions`++ - `level`: `stringOptions`, values `DEBUG`, `DEFAULT`, `WARNING`,+ `ERROR`++ - `version`: `string`++ - `traceName`: `stringOptions`++ - `userId`: `string`++ - `sessionId`: `string`++ - `tags`: `arrayOptions`++ - `metadata`: `stringObject`, requires `key`++ - `isRootObservation`: `boolean`; true when the observation has no+ parent or is explicitly marked as an application root++ - `parentObservationId`: `null`++ - `experimentId`: `stringOptions`++ - `isExperimentItemRootSpan`: `boolean`++ - `calledToolNames`: `arrayOptions`++ - `toolCalls`: `number`, the number of tool calls on the observation++ - `datasetId`: `stringOptions`. Use dataset `id` values from `GET+ /api/public/v2/datasets`, not dataset names.+ EvaluationRuleReadFilterBase:+ title: EvaluationRuleReadFilterBasetype: object+ description: Stored filter returned verbatim by evaluation-rule read endpoints.properties:- id:+ type:type: string- status:- $ref: '#/components/schemas/DatasetStatus'- input:- description: Input data for the dataset item- expectedOutput:- description: Expected output for the dataset item- metadata:- description: Metadata associated with the dataset item- sourceTraceId:+ description: >-+ Stored filter type. This is not broken down into separate public+ response types.+ column:type: string- nullable: true- description: The trace ID that sourced this dataset item- sourceObservationId:+ operator:type: string+ value:nullable: true- description: The observation ID that sourced this dataset item- datasetId:- type: string- datasetName:- type: string- createdAt:- type: string- format: date-time- updatedAt:- type: string- format: date-time- mediaReferences:- type: array- items:- $ref: '#/components/schemas/DatasetItemMediaReference'- description: >-- Resolved Langfuse media references found in input, expectedOutput,- and metadata.+ description: Stored filter value. Its shape depends on the filter type.required:- - id- - status- - input- - expectedOutput- - metadata- - sourceTraceId- - sourceObservationId- - datasetId- - datasetName- - createdAt- - updatedAt- - mediaReferences- DatasetItemMediaReference:- title: DatasetItemMediaReference+ - type+ - column+ - operator+ EvaluationRuleReadFilterWithKey:+ title: EvaluationRuleReadFilterWithKeytype: objectproperties:- field:- $ref: '#/components/schemas/DatasetItemMediaReferenceField'- description: The dataset item field containing the reference- referenceString:- type: string- description: >-- The Langfuse media reference string, e.g.- `@@@langfuseMedia:type=image/png|id=...|source=bytes@@@`- jsonPath:+ key:type: string- description: >-- JSONPath of the string holding the reference within the field, e.g.- `$['image']`- media:- $ref: '#/components/schemas/DatasetItemMediaReferenceMedia'- description: The resolved media record.+ description: Stored object or legacy filter key.required:- - field- - referenceString- - jsonPath- - media- DatasetItemMediaReferenceField:- title: DatasetItemMediaReferenceField+ - key+ allOf:+ - $ref: '#/components/schemas/EvaluationRuleReadFilterBase'+ EvaluationRuleReadFilter:+ title: EvaluationRuleReadFilter+ oneOf:+ - $ref: '#/components/schemas/EvaluationRuleReadFilterWithKey'+ - $ref: '#/components/schemas/EvaluationRuleReadFilterBase'+ description: >-+ Filter returned verbatim by evaluation-rule read endpoints.+++ Filters with a stored `key` use the keyed shape. All other filters use+ the base shape. Response filters are intentionally not broken down by+ their internal `type` value.+ PublicApiErrorCode:+ title: PublicApiErrorCodetype: stringenum:- - input- - expectedOutput- - metadata- DatasetItemMediaReferenceMedia:- title: DatasetItemMediaReferenceMedia+ - authentication_failed+ - access_denied+ - invalid_request+ - invalid_query+ - invalid_body+ - resource_not_found+ - conflict+ - rate_limited+ - method_not_allowed+ - internal_error+ description: >-+ Stable machine-readable error code returned by the evaluators and+ evaluation-rules API.+ PublicApiValidationIssue:+ title: PublicApiValidationIssuetype: object+ description: One request validation issue.properties:- mediaId:- type: string- description: The unique langfuse identifier of the media record- contentType:- type: string- description: The MIME type of the media record- contentLength:- type: integer- description: The size of the media record in bytes- url:+ code:type: string- description: The signed download URL of the media record- urlExpiry:+ description: Validator issue code.+ message:type: string- description: The expiry date and time of the download URL+ description: Human-readable explanation.+ path:+ type: array+ items: {}+ description: Path to the invalid request field.required:- - mediaId- - contentType- - contentLength- - url- - urlExpiry- DatasetRunItem:- title: DatasetRunItem+ - code+ - message+ - path+ PublicApiErrorDetails:+ title: PublicApiErrorDetailstype: object+ description: Optional structured context for validation and rate-limit errors.properties:- id:- type: string- datasetRunId:- type: string- datasetRunName:- type: string- datasetItemId:- type: string- traceId:- type: string- observationId:- type: string+ issues:+ type: array+ items:+ $ref: '#/components/schemas/PublicApiValidationIssue'nullable: true- description: The observation ID associated with this run item- createdAt:- type: string- format: date-time- updatedAt:+ description: Validation issues for an invalid request body or query.+ retryAfterSeconds:+ type: integer+ nullable: true+ description: Number of seconds to wait before retrying a rate-limited request.+ limit:+ type: integer+ nullable: true+ description: Rate-limit request allowance.+ remaining:+ type: integer+ nullable: true+ description: Remaining requests in the current rate-limit window.+ resetAt:type: stringformat: date-time- required:- - id- - datasetRunId- - datasetRunName- - datasetItemId- - traceId- - observationId- - createdAt- - updatedAt- DatasetRun:- title: DatasetRun+ nullable: true+ description: Time when the current rate-limit window resets.+ PublicApiError:+ title: PublicApiErrortype: object+ description: >-+ Standard error response for the stable evaluators and evaluation-rules+ API.+++ Use the HTTP status for the broad failure class and `code` for+ programmatic handling. `details` is included when field-level validation+ or retry information is available.properties:- id:- type: string- description: Unique identifier of the dataset run- name:- type: string- description: Name of the dataset run- description:+ message:type: string+ description: Human-readable description of the failure.+ code:+ $ref: '#/components/schemas/PublicApiErrorCode'+ description: Stable machine-readable error code.+ details:+ $ref: '#/components/schemas/PublicApiErrorDetails'nullable: true- description: Description of the run- metadata:- description: Metadata of the dataset run- datasetId:- type: string- description: Id of the associated dataset- datasetName:- type: string- description: Name of the associated dataset- createdAt:- type: string- format: date-time- description: The date and time when the dataset run was created- updatedAt:+ description: Optional structured validation or rate-limit context.+ required:+ - message+ - code+ EvaluationRuleEvaluatorAssignmentInput:+ title: EvaluationRuleEvaluatorAssignmentInput+ type: object+ properties:+ evaluatorId:type: string- format: date-time- description: The date and time when the dataset run was last updated+ description: >-+ Stable evaluator identifier. The rule automatically uses that+ evaluator's latest version.+ variableMapping:+ type: array+ items:+ $ref: '#/components/schemas/PromptVariableMappingInput'+ nullable: true+ description: >-+ Rule-specific prompt-variable mapping.+++ Set to `null` or omit to inherit the evaluator's latest default+ mapping. Code evaluators use the fixed runtime mapping and should+ use `null`.required:- - id- - name- - description- - metadata- - datasetId- - datasetName- - createdAt- - updatedAt- DatasetRunWithItems:- title: DatasetRunWithItems+ - evaluatorId+ EvaluatorAssignment:+ title: EvaluatorAssignmenttype: objectproperties:- datasetRunItems:+ evaluatorId:+ type: string+ description: Stable identifier of the evaluator associated with this rule.+ variableMapping:type: arrayitems:- $ref: '#/components/schemas/DatasetRunItem'- _deprecation:- $ref: '#/components/schemas/Deprecation'+ $ref: '#/components/schemas/PromptVariableMapping'nullable: true+ description: >-+ Stored rule-specific override, or `null` when the evaluator's latest+ default mapping is inherited. Legacy mappings are returned as an+ explicit `legacy` variant.required:- - datasetRunItems- allOf:- - $ref: '#/components/schemas/DatasetRun'- Model:- title: Model+ - evaluatorId+ - variableMapping+ EvaluationRule:+ title: EvaluationRuletype: objectdescription: >-- Model definition used for transforming usage into USD cost and/or- tokenization.--- Models can have either simple flat pricing or tiered pricing:-- - Flat pricing: Single price per usage type (legacy, but still- supported)-- - Tiered pricing: Multiple pricing tiers with conditional matching based- on usage patterns or observation attributes--- The pricing tiers approach is recommended for models with usage-based- pricing variations.+ Live evaluation rule for incoming observations.- When using tiered pricing, the flat price fields (inputPrice,- outputPrice, prices) are populated- from the default tier for backward compatibility.+ A rule determines which evaluators should be used, which observations+ should trigger scoring, how often scoring should run, and which+ observation fields should populate prompt variables.properties:id:type: string- modelName:- type: string- description: >-- Name of the model definition. If multiple with the same name exist,- they are applied in the following order: (1) custom over built-in,- (2) newest according to startTime where- model.startTime<observation.startTime- matchPattern:+ description: Stable evaluation-rule identifier.+ name:type: stringdescription: >-- Regex pattern which matches this model definition to- generation.model. Useful in case of fine-tuned models. If you want- to exact match, use `(?i)^modelname$`- startDate:- type: string- format: date-time- nullable: true- description: Apply only to generations which are newer than this ISO date.- unit:- $ref: '#/components/schemas/ModelUsageUnit'- nullable: true- description: Unit used by this model.- inputPrice:- type: number- format: double- nullable: true- description: Deprecated. See 'prices' instead. Price (USD) per input unit- outputPrice:- type: number- format: double+ Human-readable rule name. This is independent from evaluator names+ and does not need to be unique.+ createdBy:+ $ref: '#/components/schemas/Creator'nullable: true- description: Deprecated. See 'prices' instead. Price (USD) per output unit- totalPrice:+ description: User who created this rule, or `null` when no user can be resolved.+ enabled:+ type: boolean+ description: Whether live execution is enabled for this rule.+ sampling:type: numberformat: double- nullable: true- description: >-- Deprecated. See 'prices' instead. Price (USD) per total unit. Cannot- be set if input or output price is set.- tokenizerId:- type: string- nullable: true- description: >-- Optional. Tokenizer to be applied to observations which match to- this model. See docs for more details.- tokenizerConfig:- description: >-- Optional. Configuration for the selected tokenizer. Needs to be- JSON. See docs for more details.- isLangfuseManaged:- type: boolean- createdAt:- type: string- format: date-time- description: Timestamp when the model was created- prices:- type: object- additionalProperties:- $ref: '#/components/schemas/ModelPrice'- description: >-- Deprecated. Use 'pricingTiers' instead for models with usage-based- pricing variations.--- This field shows prices by usage type from the default pricing tier.- Maintained for backward compatibility.+ description: |-+ Fraction of matching observations that should be evaluated.- If the model uses tiered pricing, this field will be populated from- the default tier's prices.- pricingTiers:+ Must be between `0` and `1`.+ - `1` evaluates every matching observation.+ - `0.25` evaluates approximately 25% of matching observations.+ filter:type: arrayitems:- $ref: '#/components/schemas/PricingTier'+ $ref: '#/components/schemas/EvaluationRuleReadFilter'description: >-- Array of pricing tiers with conditional pricing based on usage- thresholds or observation attributes.--- Pricing tiers enable accurate cost tracking for models that charge- different rates based on usage patterns-- or request attributes (e.g., high-volume usage, large context- windows, cached tokens, or service tiers).--- Each model must have exactly one default tier (isDefault=true,- priority=0) that serves as a fallback.-- Additional conditional tiers can be defined with specific matching- criteria.--- If this array is empty, the model uses legacy flat pricing from the- inputPrice/outputPrice/totalPrice fields.+ List of stored filter conditions returned verbatim. Filters with a+ `key` use the keyed response shape; all others use the base shape.+ These response shapes are not broken down by internal filter type.+ An empty list matches every incoming object.+ evaluatorAssignments:+ type: array+ items:+ $ref: '#/components/schemas/EvaluatorAssignment'+ description: Evaluators attached to this rule in deterministic assignment order.+ createdAt:+ type: string+ format: date-time+ description: Timestamp when the evaluation rule was created.+ updatedAt:+ type: string+ format: date-time+ description: Timestamp when the evaluation rule was last updated.required:- id- - modelName- - matchPattern- - startDate- - inputPrice- - outputPrice- - totalPrice- - tokenizerId- - tokenizerConfig- - isLangfuseManaged+ - name+ - createdBy+ - enabled+ - sampling+ - filter+ - evaluatorAssignments- createdAt- - prices- - pricingTiers- ModelPrice:- title: ModelPrice+ - updatedAt+ CreateEvaluationRuleRequest:+ title: CreateEvaluationRuleRequesttype: objectproperties:- price:+ name:+ type: string+ description: >-+ Human-readable rule name. Names are not identifiers and do not need+ to be unique.+ enabled:+ type: boolean+ description: >-+ Whether live execution should start immediately. Enabled rules+ require at least one evaluator assignment.+ sampling:type: numberformat: double+ nullable: true+ description: >-+ Fraction of matching observations to evaluate. Omit this field to+ use the default of `1`, which evaluates every match. `null` is not+ accepted.+ filter:+ type: array+ items:+ $ref: '#/components/schemas/EvaluationRuleFilter'+ nullable: true+ description: >-+ Conditions used to select observations. Defaults to an empty list,+ which matches every incoming observation.+ evaluatorAssignments:+ type: array+ items:+ $ref: '#/components/schemas/EvaluationRuleEvaluatorAssignmentInput'+ description: >-+ Evaluators to attach to this rule. Disabled rules may use an empty+ list as a draft.required:- - price- PricingTierCondition:- title: PricingTierCondition- oneOf:- - $ref: '#/components/schemas/PricingTierUsageCondition'- - $ref: '#/components/schemas/PricingTierAttributeCondition'- description: >-- Condition for matching a pricing tier against usage details or- observation attributes.--- Usage-detail conditions treat usageDetailPattern as a regex, sum all- matching usage values, and compare the sum to the numeric value.- Model-parameter and metadata conditions match an exact top-level key- against one or more string values.- PricingTierUsageCondition:- title: PricingTierUsageCondition+ - name+ - enabled+ - evaluatorAssignments+ UpdateEvaluationRuleRequest:+ title: UpdateEvaluationRuleRequesttype: object- description: Condition that sums usage details whose keys match a regex.+ description: At least one field must be provided.properties:- usageDetailPattern:+ name:type: string- description: Regex pattern matched against usage detail keys.- operator:- $ref: '#/components/schemas/PricingTierOperator'- value:+ nullable: true+ description: New human-readable rule name.+ enabled:+ type: boolean+ nullable: true+ description: New desired live-execution state.+ sampling:type: numberformat: double- description: Numeric threshold for the summed matching usage values.- caseSensitive:- type: boolean- description: Whether the usage-detail regex is case-sensitive.- required:- - usageDetailPattern- - operator- - value- - caseSensitive- PricingTierAttributeCondition:- title: PricingTierAttributeCondition- type: object- description: >-- Condition that matches any configured value for a top-level observation- attribute.- properties:- source:- $ref: '#/components/schemas/PricingTierAttributeSource'- description: Observation attribute object evaluated by this condition.- key:- type: string- description: Exact top-level attribute key.- operator:- type: string- const: in- description: Membership operator.- values:+ nullable: true+ description: >-+ New fraction of matching observations to evaluate. Omit to keep the+ current value.+ filter:+ type: array+ items:+ $ref: '#/components/schemas/EvaluationRuleFilter'+ nullable: true+ description: >-+ Complete replacement filter list. An empty list matches every+ incoming observation.+ evaluatorAssignments:+ type: array+ items:+ $ref: '#/components/schemas/EvaluationRuleEvaluatorAssignmentInput'+ nullable: true+ description: >-+ Complete replacement assignment list. An empty list disables the+ rule.+ EvaluationRulesPage:+ title: EvaluationRulesPage+ type: object+ properties:+ data:type: arrayitems:- type: string- description: Accepted string attribute values. At least one value is required.+ $ref: '#/components/schemas/EvaluationRule'+ description: Evaluation rules for this page.+ meta:+ $ref: '#/components/schemas/CursorMeta'+ description: Cursor pagination metadata.required:- - source- - key- - operator- - values- PricingTierAttributeSource:- title: PricingTierAttributeSource- type: string- enum:- - model_parameters- - metadata- PricingTierConditionInput:- title: PricingTierConditionInput- oneOf:- - $ref: '#/components/schemas/PricingTierUsageConditionInput'- - $ref: '#/components/schemas/PricingTierAttributeCondition'- description: Condition input for matching a pricing tier.- PricingTierUsageConditionInput:- title: PricingTierUsageConditionInput+ - data+ - meta+ DeletedEvaluationRule:+ title: DeletedEvaluationRuletype: object- description: Input condition that sums usage details whose keys match a regex.+ description: Confirmation returned after successful evaluation-rule deletion.properties:- usageDetailPattern:+ id:+ type: string+ description: Identifier of the deleted evaluation rule.+ required:+ - id+ Creator:+ title: Creator+ type: object+ description: User who created the resource, or `null` when no user can be resolved.+ properties:+ id:+ type: string+ description: User identifier.+ name:type: string- description: Regex pattern matched against usage detail keys.- operator:- $ref: '#/components/schemas/PricingTierOperator'- value:- type: number- format: double- description: Numeric threshold for the summed matching usage values.- caseSensitive:- type: booleannullable: true- description: Whether the usage-detail regex is case-sensitive. Defaults to false.+ description: User name, or `null` when unavailable.required:- - usageDetailPattern- - operator- - value- PricingTier:- title: PricingTier+ - id+ - name+ EvaluatorModelConfig:+ title: EvaluatorModelConfigtype: objectdescription: >-- Pricing tier definition with conditional pricing based on usage- thresholds or observation attributes.--- Pricing tiers enable accurate cost tracking for LLM providers that- charge different rates based on usage patterns or request attributes.-- For example, some providers charge higher rates when context size- exceeds certain thresholds or when an accelerated service tier is- selected.--- How tier matching works:-- 1. Tiers are evaluated in ascending priority order (priority 1 before- priority 2, etc.)-- 2. The first tier where ALL conditions match is selected-- 3. If no conditional tiers match, the default tier is used as a fallback-- 4. The default tier has priority 0 and no conditions--- Why priorities matter:-- - Lower priority numbers are evaluated first, allowing you to define- specific cases before general ones+ Explicit model configuration for an evaluator.- - Example: Priority 1 for "high usage" (>200K tokens), Priority 2 for- "medium usage" (>100K tokens), Priority 0 for default-- - Without proper ordering, a less specific condition might match before- a more specific one+ If the complete `modelConfig` is `null`, Langfuse uses the project's+ default evaluation model.- Every model must have exactly one default tier to ensure cost- calculation always succeeds.+ If provided, the model must be available to the project when the+ evaluator or evaluation rule is enabled.properties:- id:- type: string- description: Unique identifier for the pricing tier- name:+ provider:type: stringdescription: >-- Name of the pricing tier for display and identification purposes.+ Provider identifier, for example `openai` or `anthropic`.- Examples: "Standard", "High Volume Tier", "Large Context", "Extended- Context Tier"- isDefault:- type: boolean+ To discover valid values for the current project, call `GET+ /api/public/llm-connections` and use one of the returned `provider`+ values.+ model:+ type: stringdescription: >-- Whether this is the default tier. Every model must have exactly one- default tier with priority 0 and no conditions.--- The default tier serves as a fallback when no conditional tiers- match, ensuring cost calculation always succeeds.-- It typically represents the base pricing for standard usage- patterns.- priority:+ Model identifier exposed by the provider, for example `gpt-4.1-mini`.+ required:+ - provider+ - model+ EvaluatorChatMessage:+ title: EvaluatorChatMessage+ type: object+ description: One user chat message in an evaluator prompt.+ properties:+ role:+ type: string+ const: user+ content:+ type: string+ description: Message content. Evaluator variables use `{{variable}}` syntax.+ required:+ - role+ - content+ EvaluatorChatPrompt:+ title: EvaluatorChatPrompt+ type: array+ items:+ $ref: '#/components/schemas/EvaluatorChatMessage'+ description: A list containing exactly one user chat message.+ EvaluatorChatPromptInput:+ title: EvaluatorChatPromptInput+ oneOf:+ - type: string+ - $ref: '#/components/schemas/EvaluatorChatPrompt'+ description: >-+ A user prompt string, or a list containing exactly one user chat message.+ EvaluatorVersionBase:+ title: EvaluatorVersionBase+ type: object+ properties:+ id:+ type: string+ description: Stable identifier of this evaluator version.+ version:type: integer+ description: Monotonically increasing evaluator version number.+ createdAt:+ type: string+ format: date-time+ description: Timestamp when this evaluator version was created.+ createdBy:+ $ref: '#/components/schemas/Creator'+ nullable: truedescription: >-- Priority for tier matching evaluation. Lower numbers = higher- priority (evaluated first).--- The default tier must always have priority 0. Conditional tiers- should have priority 1, 2, 3, etc.--- Example ordering:-- - Priority 0: Default tier (no conditions, always matches as- fallback)-- - Priority 1: High usage tier (e.g., >200K tokens)-- - Priority 2: Medium usage tier (e.g., >100K tokens)--- This ensures more specific conditions are checked before general- ones.- conditions:+ User who created this version, or `null` when no user can be+ resolved.+ required:+ - id+ - version+ - createdAt+ - createdBy+ LlmAsJudgeEvaluatorVersion:+ title: LlmAsJudgeEvaluatorVersion+ type: object+ properties:+ type:+ type: string+ const: llm_as_judge+ description: Evaluator type.+ prompt:+ $ref: '#/components/schemas/EvaluatorChatPrompt'+ description: The single user chat message used during evaluation.+ variables:type: arrayitems:- $ref: '#/components/schemas/PricingTierCondition'+ type: stringdescription: >-- Array of conditions that must ALL be met for this tier to match (AND- logic).--- The default tier must have an empty conditions array. Conditional- tiers should have one or more conditions-- that define when this tier's pricing applies.+ Variables extracted from the prompt and available for+ evaluation-rule mappings.- Multiple conditions enable complex matching scenarios (e.g., "high- input tokens AND low output tokens").- prices:- type: object- additionalProperties:- type: number- format: double+ Every variable must be mapped exactly once when a rule provides an+ explicit mapping.+ variableMapping:+ type: array+ items:+ $ref: '#/components/schemas/PromptVariableMappingRead'+ nullable: truedescription: >-- Prices (USD) by usage type for this tier.--- Common usage types: "input", "output", "total", "request", "image"-- Prices are specified in USD per unit (e.g., per token, per request,- per second).+ Default variable mapping for this evaluator version, or `null` when+ no default is configured.+ modelConfig:+ $ref: '#/components/schemas/EvaluatorModelConfig'+ nullable: true+ description: >-+ Explicit model configuration, or `null` when the project's default+ evaluation model is used.+ outputDefinition:+ $ref: '#/components/schemas/PublicEvaluatorOutputDefinition'+ description: >-+ Structured output schema returned by this evaluator.- Example: {"input": 0.000003, "output": 0.000015} means $3 per- million input tokens and $15 per million output tokens.+ Responses include `dataType` and omit the internal output-definition+ persistence version.+ required:+ - type+ - prompt+ - variables+ - variableMapping+ - modelConfig+ - outputDefinition+ allOf:+ - $ref: '#/components/schemas/EvaluatorVersionBase'+ CodeEvaluatorVersion:+ title: CodeEvaluatorVersion+ type: object+ properties:+ type:+ type: string+ const: code+ description: Evaluator type.+ sourceCode:+ type: string+ description: Source code executed for each matched observation.+ sourceCodeLanguage:+ $ref: '#/components/schemas/CodeEvaluatorSourceCodeLanguage'+ description: Runtime language used to execute the source code.+ required:+ - type+ - sourceCode+ - sourceCodeLanguage+ allOf:+ - $ref: '#/components/schemas/EvaluatorVersionBase'+ EvaluatorVersion:+ title: EvaluatorVersion+ oneOf:+ - type: object+ allOf:+ - type: object+ properties:+ type:+ type: string+ enum:+ - llm_as_judge+ - $ref: '#/components/schemas/LlmAsJudgeEvaluatorVersion'+ required:+ - type+ - type: object+ allOf:+ - type: object+ properties:+ type:+ type: string+ enum:+ - code+ - $ref: '#/components/schemas/CodeEvaluatorVersion'+ required:+ - type+ EvaluationRuleAssignment:+ title: EvaluationRuleAssignment+ type: object+ description: Evaluation-rule assignment associated with this evaluator.+ properties:+ evaluationRuleId:+ type: string+ description: Stable identifier of the assigned evaluation rule.+ variableMappingOverride:+ type: array+ items:+ $ref: '#/components/schemas/PromptVariableMapping'+ nullable: true+ description: >-+ Rule-specific variable mapping override. Omitted when the+ evaluator's latest default mapping is inherited. Legacy mappings use+ the explicit `legacy` variant.required:- - id- - name- - isDefault- - priority- - conditions- - prices- PricingTierInput:- title: PricingTierInput+ - evaluationRuleId+ EvaluatorBase:+ title: EvaluatorBasetype: object- description: >-- Input schema for creating a pricing tier. The tier ID will be- automatically generated server-side.--- When creating a model with pricing tiers:-- - Exactly one tier must have isDefault=true (the fallback tier)-- - The default tier must have priority=0 and conditions=[]-- - All tier names and priorities must be unique within the model-- - Each tier must define at least one price--- See PricingTier for detailed information about how tiers work and why- they're useful.properties:+ id:+ type: string+ description: Stable identifier of this evaluator across all versions.name:type: stringdescription: >-- Name of the pricing tier for display and identification purposes.--- Must be unique within the model. Common patterns: "Standard", "High- Volume Tier", "Extended Context"- isDefault:- type: boolean+ Human-readable evaluator name. Names are not identifiers and do not+ need to be unique.+ description:+ type: string+ nullable: true+ description: Optional human-readable evaluator description.+ createdBy:+ $ref: '#/components/schemas/Creator'nullable: truedescription: >-- Whether this is the default tier. Exactly one tier per model must be- marked as default.--- Requirements for default tier:-- - Must have isDefault=true-- - Must have priority=0-- - Must have empty conditions array (conditions=[])--- The default tier acts as a fallback when no conditional tiers match.- Defaults to false when omitted.- priority:- type: integer+ User who created this evaluator, or `null` when no user can be+ resolved.+ status:+ $ref: '#/components/schemas/EvaluatorStatus'description: >-- Priority for tier matching evaluation. Lower numbers = higher- priority (evaluated first).--- Must be unique within the model. The default tier must have- priority=0.-- Conditional tiers should use priority 1, 2, 3, etc. based on their- specificity.- conditions:+ Effective evaluator status after Langfuse validates its runtime+ configuration.+ pausedAt:+ type: string+ format: date-time+ nullable: true+ description: Timestamp when the evaluator was paused, otherwise `null`.+ pausedReason:+ type: string+ nullable: true+ description: Machine-readable reason when `status=paused`, otherwise `null`.+ pausedMessage:+ type: string+ nullable: true+ description: Human-readable explanation when `status=paused`, otherwise `null`.+ evaluationRuleAssignments:type: arrayitems:- $ref: '#/components/schemas/PricingTierConditionInput'+ $ref: '#/components/schemas/EvaluationRuleAssignment'description: >-- Array of conditions that must ALL be met for this tier to match (AND- logic).--- The default tier must have an empty array (conditions=[]).-- Conditional tiers should define one or more conditions that specify- when this tier's pricing applies.--- Conditions can compare summed matching usage details to a numeric- threshold, or exactly match a top-level model parameter or metadata- value.- prices:- type: object- additionalProperties:- type: number- format: double+ All modern and legacy evaluation-rule assignments in+ newest-assignment-first order, including each assigned rule ID and+ its stored mapping.+ createdAt:+ type: string+ format: date-time+ description: Timestamp when the evaluator was created.+ updatedAt:+ type: string+ format: date-time+ description: Timestamp when the evaluator was last updated.+ versionId:+ type: string+ description: Stable identifier of the latest evaluator version.+ version:+ type: integer+ description: Monotonically increasing latest evaluator version number.+ versionCreatedAt:+ type: string+ format: date-time+ description: Timestamp when the latest evaluator version was created.+ versionCreatedBy:+ $ref: '#/components/schemas/Creator'+ nullable: truedescription: >-- Prices (USD) by usage type for this tier. At least one price must be- defined.--- Common usage types: "input", "output", "total", "request", "image"-- Prices are in USD per unit (e.g., per token).--- Example: {"input": 0.000003, "output": 0.000015} represents $3 per- million input tokens and $15 per million output tokens.+ User who created the latest evaluator version, or `null` when no+ user can be resolved.required:+ - id- name- - priority- - conditions- - prices- PricingTierOperator:- title: PricingTierOperator- type: string- enum:- - gt- - gte- - lt- - lte- - eq- - neq- description: Comparison operators for pricing tier conditions- ModelUsageUnit:- title: ModelUsageUnit- type: string- enum:- - CHARACTERS- - TOKENS- - MILLISECONDS- - SECONDS- - IMAGES- - REQUESTS- description: Unit of usage in Langfuse- ObservationLevel:- title: ObservationLevel- type: string- enum:- - DEBUG- - DEFAULT- - WARNING- - ERROR- MapValue:- title: MapValue- oneOf:- - type: string- nullable: true- - type: integer- nullable: true- - type: number- format: float- nullable: true- - type: boolean- nullable: true- - type: array+ - description+ - createdBy+ - status+ - pausedAt+ - pausedReason+ - pausedMessage+ - evaluationRuleAssignments+ - createdAt+ - updatedAt+ - versionId+ - version+ - versionCreatedAt+ - versionCreatedBy+ LlmAsJudgeEvaluator:+ title: LlmAsJudgeEvaluator+ type: object+ properties:+ type:+ type: string+ const: llm_as_judge+ description: Evaluator type.+ prompt:+ $ref: '#/components/schemas/EvaluatorChatPrompt'+ description: The single user chat message used by the latest evaluator version.+ variables:+ type: arrayitems:type: string+ description: >-+ Variables extracted from the latest prompt and available for+ evaluation-rule mappings.+ variableMapping:+ type: array+ items:+ $ref: '#/components/schemas/PromptVariableMappingRead'nullable: true- CommentObjectType:- title: CommentObjectType- type: string- enum:- - TRACE- - OBSERVATION- - SESSION- - PROMPT- DatasetStatus:- title: DatasetStatus- type: string- enum:- - ACTIVE- - ARCHIVED- ScoreSource:- title: ScoreSource- type: string- enum:- - ANNOTATION- - API- - EVAL- ScoreConfigDataType:- title: ScoreConfigDataType- type: string- enum:- - NUMERIC- - BOOLEAN- - CATEGORICAL- - TEXT- ScoreDataType:- title: ScoreDataType+ description: >-+ Default variable mapping for the latest version, or `null` when no+ default is configured.+ modelConfig:+ $ref: '#/components/schemas/EvaluatorModelConfig'+ nullable: true+ description: >-+ Explicit model configuration for the latest version, or `null` when+ the project's default evaluation model is used.+ outputDefinition:+ $ref: '#/components/schemas/PublicEvaluatorOutputDefinition'+ description: Structured output schema returned by the latest evaluator version.+ required:+ - type+ - prompt+ - variables+ - variableMapping+ - modelConfig+ - outputDefinition+ allOf:+ - $ref: '#/components/schemas/EvaluatorBase'+ CodeEvaluator:+ title: CodeEvaluator+ type: object+ properties:+ type:+ type: string+ const: code+ description: Evaluator type.+ sourceCode:+ type: string+ description: Source code executed by the latest evaluator version.+ sourceCodeLanguage:+ $ref: '#/components/schemas/CodeEvaluatorSourceCodeLanguage'+ description: Runtime language used to execute the latest source code.+ required:+ - type+ - sourceCode+ - sourceCodeLanguage+ allOf:+ - $ref: '#/components/schemas/EvaluatorBase'+ Evaluator:+ title: Evaluator+ oneOf:+ - type: object+ allOf:+ - type: object+ properties:+ type:+ type: string+ enum:+ - llm_as_judge+ - $ref: '#/components/schemas/LlmAsJudgeEvaluator'+ required:+ - type+ - type: object+ allOf:+ - type: object+ properties:+ type:+ type: string+ enum:+ - code+ - $ref: '#/components/schemas/CodeEvaluator'+ required:+ - type+ description: >-+ One evaluator that can be used for scoring.+++ An evaluator describes **how** to score data. Associated evaluation+ rules describe **which** live objects should be evaluated.+ EvaluatorStatus:+ title: EvaluatorStatustype: stringenum:- - NUMERIC- - BOOLEAN- - CATEGORICAL- - CORRECTION- - TEXT- DeleteDatasetItemResponse:- title: DeleteDatasetItemResponse- type: object- properties:- message:- type: string- description: Success message after deletion- required:- - message- CreateDatasetItemRequest:- title: CreateDatasetItemRequest+ - active+ - paused+ description: >-+ Effective evaluator runtime status.+++ - `active`: the evaluator can run.++ - `paused`: Langfuse paused execution until the underlying issue is+ resolved.+ CreateLlmAsJudgeEvaluatorRequest:+ title: CreateLlmAsJudgeEvaluatorRequesttype: objectproperties:- datasetName:- type: string- input:- nullable: true- expectedOutput:- nullable: true- metadata:- nullable: true- sourceTraceId:+ name:type: string- nullable: true- sourceObservationId:+ description: >-+ Human-readable evaluator name. Names are not identifiers and do not+ need to be unique.+ description:type: stringnullable: true- id:+ description: Optional human-readable evaluator description.+ type:type: string+ const: llm_as_judge+ description: Evaluator type.+ prompt:+ $ref: '#/components/schemas/EvaluatorChatPromptInput'+ description: >-+ User prompt string shortcut or a list containing exactly one user+ chat message.+ modelConfig:+ $ref: '#/components/schemas/EvaluatorModelConfig'nullable: truedescription: >-- Dataset items are upserted on their id. Id needs to be unique- (project-level), cannot be reused across datasets, and must be at- most 255 characters.- status:- $ref: '#/components/schemas/DatasetStatus'+ Explicit model configuration. Set to `null` or omit to use the+ project's default evaluation model.+ variableMapping:+ type: array+ items:+ $ref: '#/components/schemas/PromptVariableMappingInput'nullable: true- description: Defaults to ACTIVE for newly created items+ description: >-+ Default prompt-variable mapping, or `null` when no default is+ configured.+ outputDefinition:+ $ref: '#/components/schemas/EvaluatorOutputDefinition'+ description: >-+ Structured output schema returned by this evaluator. Do not include+ an internal persistence version.required:- - datasetName- PaginatedDatasetItems:- title: PaginatedDatasetItems+ - name+ - type+ - prompt+ - outputDefinition+ CreateCodeEvaluatorRequest:+ title: CreateCodeEvaluatorRequesttype: objectproperties:- data:- type: array- items:- $ref: '#/components/schemas/DatasetItem'- meta:- $ref: '#/components/schemas/utilsMetaResponse'+ name:+ type: string+ description: >-+ Human-readable evaluator name. Names are not identifiers and do not+ need to be unique.+ description:+ type: string+ nullable: true+ description: Optional human-readable evaluator description.+ type:+ type: string+ const: code+ description: Evaluator type.+ sourceCode:+ type: string+ description: Source code executed for each matched observation.+ sourceCodeLanguage:+ $ref: '#/components/schemas/CodeEvaluatorSourceCodeLanguage'+ description: Runtime language used to execute the source code.required:- - data- - meta- CreateDatasetRunItemRequest:- title: CreateDatasetRunItemRequest+ - name+ - type+ - sourceCode+ - sourceCodeLanguage+ CreateEvaluatorRequest:+ title: CreateEvaluatorRequest+ oneOf:+ - type: object+ allOf:+ - type: object+ properties:+ type:+ type: string+ enum:+ - llm_as_judge+ - $ref: '#/components/schemas/CreateLlmAsJudgeEvaluatorRequest'+ required:+ - type+ - type: object+ allOf:+ - type: object+ properties:+ type:+ type: string+ enum:+ - code+ - $ref: '#/components/schemas/CreateCodeEvaluatorRequest'+ required:+ - type+ description: >-+ Initial evaluator definition with metadata and definition fields at the+ same level. The returned evaluator starts at version `1`.+ UpdateEvaluatorMetadataRequest:+ title: UpdateEvaluatorMetadataRequesttype: object+ description: Metadata-only update. At least one field must be provided.properties:- runName:- type: string- runDescription:+ name:type: stringnullable: true- description: Description of the run. If run exists, description will be updated.- metadata:+ description: New human-readable evaluator name.+ description:+ type: stringnullable: true- description: Metadata of the dataset run, updates run if run already exists- datasetItemId:+ description: New description. Set to `null` to clear it.+ UpdateLlmAsJudgeEvaluatorRequest:+ title: UpdateLlmAsJudgeEvaluatorRequest+ type: object+ properties:+ name:type: string- observationId:+ nullable: true+ description: New human-readable evaluator name.+ description:type: stringnullable: true- traceId:+ description: New description. Set to `null` to clear it.+ type:type: string+ const: llm_as_judge+ description: Evaluator type. The type of an existing evaluator cannot change.+ prompt:+ $ref: '#/components/schemas/EvaluatorChatPromptInput'+ description: >-+ Complete replacement user prompt string or a list containing exactly+ one user chat message.+ modelConfig:+ $ref: '#/components/schemas/EvaluatorModelConfig'nullable: truedescription: >-- traceId should always be provided. For compatibility with older SDK- versions it can also be inferred from the provided observationId.- datasetVersion:- type: string- format: date-time+ Explicit model configuration. Set to `null` or omit to use the+ project's default evaluation model.+ variableMapping:+ type: array+ items:+ $ref: '#/components/schemas/PromptVariableMappingInput'nullable: truedescription: >-- ISO 8601 timestamp (RFC 3339, Section 5.6) in UTC (e.g.,- "2026-01-21T14:35:42Z").-- Specifies the dataset version to use for this experiment run.-- If provided, the experiment will use dataset items as they existed- at or before this timestamp.-- If not provided, uses the latest version of dataset items.- createdAt:+ Complete replacement default variable mapping, or `null` when no+ default is configured.+ outputDefinition:+ $ref: '#/components/schemas/EvaluatorOutputDefinition'+ description: >-+ Complete replacement output schema. Do not include an internal+ persistence version.+ required:+ - type+ - prompt+ - outputDefinition+ UpdateCodeEvaluatorRequest:+ title: UpdateCodeEvaluatorRequest+ type: object+ properties:+ name:type: string- format: date-timenullable: true- description: >-- Optional timestamp to set the createdAt field of the dataset run- item. If not provided or null, defaults to current timestamp.+ description: New human-readable evaluator name.+ description:+ type: string+ nullable: true+ description: New description. Set to `null` to clear it.+ type:+ type: string+ const: code+ description: Evaluator type. The type of an existing evaluator cannot change.+ sourceCode:+ type: string+ description: Complete replacement source code.+ sourceCodeLanguage:+ $ref: '#/components/schemas/CodeEvaluatorSourceCodeLanguage'+ description: Runtime language used to execute the source code.required:- - runName- - datasetItemId- PaginatedDatasetRunItems:- title: PaginatedDatasetRunItems+ - type+ - sourceCode+ - sourceCodeLanguage+ UpdateEvaluatorRequest:+ title: UpdateEvaluatorRequest+ oneOf:+ - $ref: '#/components/schemas/UpdateEvaluatorMetadataRequest'+ - $ref: '#/components/schemas/UpdateLlmAsJudgeEvaluatorRequest'+ - $ref: '#/components/schemas/UpdateCodeEvaluatorRequest'+ description: >-+ Metadata-only update or complete flattened definition replacement. At+ least one field must be provided.+ EvaluatorsPage:+ title: EvaluatorsPagetype: objectproperties:data:type: arrayitems:- $ref: '#/components/schemas/DatasetRunItem'+ $ref: '#/components/schemas/Evaluator'+ description: Evaluators for this page.meta:- $ref: '#/components/schemas/utilsMetaResponse'- _deprecation:- $ref: '#/components/schemas/Deprecation'- nullable: true+ $ref: '#/components/schemas/CursorMeta'+ description: Cursor pagination metadata.required:- data- meta- PaginatedDatasets:- title: PaginatedDatasets+ EvaluatorVersionsPage:+ title: EvaluatorVersionsPagetype: objectproperties:data:type: arrayitems:- $ref: '#/components/schemas/Dataset'+ $ref: '#/components/schemas/EvaluatorVersion'+ description: Evaluator versions for this page in newest-first order.meta:- $ref: '#/components/schemas/utilsMetaResponse'+ $ref: '#/components/schemas/CursorMeta'+ description: Cursor pagination metadata.required:- data- meta- CreateDatasetRequest:- title: CreateDatasetRequest+ CursorMeta:+ title: CursorMetatype: objectproperties:- name:- type: string- description:+ cursor:type: stringnullable: true- metadata:- nullable: true- inputSchema:- nullable: true- description: >-- JSON Schema for validating dataset item inputs. When set, all new- and existing dataset items will be validated against this schema.- expectedOutputSchema:- nullable: true- description: >-- JSON Schema for validating dataset item expected outputs. When set,- all new and existing dataset items will be validated against this- schema.- required:- - name- PaginatedDatasetRuns:- title: PaginatedDatasetRuns- type: object- properties:- data:- type: array- items:- $ref: '#/components/schemas/DatasetRun'- meta:- $ref: '#/components/schemas/utilsMetaResponse'- _deprecation:- $ref: '#/components/schemas/Deprecation'- nullable: true- required:- - data- - meta- DeleteDatasetRunResponse:- title: DeleteDatasetRunResponse+ description: Opaque cursor for the next page. Omitted when there is no next page.+ DeletedEvaluator:+ title: DeletedEvaluatortype: object+ description: Confirmation returned after successful evaluator deletion.properties:- message:+ id:type: string+ description: Identifier of the deleted evaluator.required:- - message+ - idExperimentsResponse:title: ExperimentsResponsetype: object@@ -14481,7 +17186,7 @@ components:- llm_as_judge- codedescription: |-- The evaluator engine type.+ The evaluator type.The unstable public API supports LLM-as-a-judge and code evaluators.unstableCodeEvaluatorSourceCodeLanguage:@@ -14528,8 +17233,8 @@ components:- `paused`: enabled, but Langfuse has blocked execution until theunderlying issue is resolved.- unstableEvaluationRuleMappingSource:- title: unstableEvaluationRuleMappingSource+ unstablePromptVariableMappingSource:+ title: unstablePromptVariableMappingSourcetype: stringenum:- input@@ -15072,11 +17777,11 @@ components:required:- column- operator- unstableEvaluationRuleMapping:- title: unstableEvaluationRuleMapping+ unstablePromptVariableMappingInput:+ title: unstablePromptVariableMappingInputtype: objectdescription: >-- Maps one evaluator variable to one source field from the target object.+ Connects one prompt variable to the available evaluation data.Manual mappings are used for `llm_as_judge` evaluators. `code`@@ -15107,10 +17812,10 @@ components:- `invalid_variable_mapping`: the variable name is unknown for thisevaluator, or the selected `source` is not valid for the chosen `target`- - `missing_variable_mapping`: one or more LLM-as-judge evaluator- variables are not mapped yet+ - `missing_variable_mapping`: one or more LLM-as-judge prompt variables+ are not mapped yet- - `duplicate_variable_mapping`: the same evaluator variable appears more+ - `duplicate_variable_mapping`: the same prompt variable appears morethan once- `invalid_json_path`: the JSONPath expression is malformed. Remove it@@ -15125,7 +17830,7 @@ components:Example: for the prompt `Judge {{input}} against {{output}}`, use`input` and `output`.source:- $ref: '#/components/schemas/unstableEvaluationRuleMappingSource'+ $ref: '#/components/schemas/unstablePromptVariableMappingSource'description: >-Source field that should populate the prompt variable.@@ -15154,34 +17859,19 @@ components:required:- variable- source- unstableEvaluationRuleReadMapping:- title: unstableEvaluationRuleReadMapping+ unstablePromptVariableMappingRead:+ title: unstablePromptVariableMappingReadtype: object- description: >-- Maps an evaluator variable to target data, as returned by read- endpoints.--- Reads are more permissive than writes in two ways, so that data- configured before a-- migration stays readable and repairable:-- - `source` is `null` when a mapping was never fully configured. Set a- concrete `source` to fix it.-- - every `source` value is allowed regardless of the rule `target`,- because an evaluator's- default mapping is stored independently of the rules that use it.+ description: |-+ Connects one prompt variable to data from a target.- Requests always require a concrete `source` that is valid for the- selected `target`.+ `source` is `null` when the mapping is incomplete.properties:variable:type: stringdescription: Prompt variable name without braces.source:- $ref: '#/components/schemas/unstableEvaluationRuleMappingSource'+ $ref: '#/components/schemas/unstablePromptVariableMappingSource'nullable: truedescription: >-Source field populating the variable, or `null` when the mapping is@@ -16198,7 +18888,7 @@ components:- how often scoring should run- - which target fields should populate each evaluator variable+ - which target fields should populate each prompt variable- whether the deployment is active, inactive, or paused@@ -16303,7 +18993,7 @@ components:mapping:type: arrayitems:- $ref: '#/components/schemas/unstableEvaluationRuleReadMapping'+ $ref: '#/components/schemas/unstablePromptVariableMappingRead'description: >-Deprecated compatibility alias containing the effective mapping for`evaluators[0]`.@@ -16318,13 +19008,13 @@ components:title: unstableLegacyEvaluationRuletype: objectdescription: >-- Legacy trace- or dataset-level evaluation rule returned by list and get- for migration.+ **Deprecated:** Legacy trace- or dataset-level evaluation rule returned+ by list and get for migration.This resource is read-only through the unstable public API. Its mappingpreserves the trace, dataset item, or named observation selected for- each evaluator variable.+ each prompt variable.properties:evaluators:type: array@@ -16352,7 +19042,7 @@ components:mapping:type: arrayitems:- $ref: '#/components/schemas/unstableLegacyEvaluationRuleMapping'+ $ref: '#/components/schemas/unstableLegacyPromptVariableMapping'description: >-Stored variable mappings, including the trace, dataset item, ornamed observation selected for each variable.@@ -16412,7 +19102,7 @@ components:`filter[].value`- for `llm_as_judge`, fetch or inspect the evaluator first and provide a- complete variable mapping for every evaluator variable+ complete mapping for every prompt variable- for `code`, do not send variables or mappings; Langfuse stores thefixed code runtime mapping automatically@@ -16508,15 +19198,12 @@ components:mapping:type: arrayitems:- $ref: '#/components/schemas/unstableEvaluationRuleMapping'- description: >-+ $ref: '#/components/schemas/unstablePromptVariableMappingInput'+ description: |-LLM-as-judge variable mappings.-- Every evaluator variable must appear exactly once.-- Build this list from the evaluator `variables` array returned by the- evaluator endpoints.+ Every prompt variable must appear exactly once.+ Build this list from the evaluator's `variables` array.required:- name- evaluator@@ -16599,7 +19286,7 @@ components:assignment.- If you change to an LLM-as-judge `evaluator`, send a fresh `mapping`- unless you are certain the existing mapping still matches the evaluator+ unless you are certain the existing mapping still matches the promptvariables.- If you change `target` for an LLM-as-judge rule, usually send both@@ -16671,7 +19358,7 @@ components:mapping:type: arrayitems:- $ref: '#/components/schemas/unstableEvaluationRuleMapping'+ $ref: '#/components/schemas/unstablePromptVariableMappingInput'nullable: truedescription: >-Updated LLM-as-judge variable mappings.@@ -16722,7 +19409,7 @@ components:type:$ref: '#/components/schemas/unstableLlmAsJudgeEvaluatorType'nullable: true- description: Evaluator engine type. Defaults to `llm_as_judge` when omitted.+ description: Evaluator type. Defaults to `llm_as_judge` when omitted.required:- nameunstableCodeEvaluationRuleEvaluatorReference:@@ -16763,7 +19450,7 @@ components:description: Evaluator family name.type:$ref: '#/components/schemas/unstableEvaluatorType'- description: Evaluator engine type.+ description: Evaluator type.required:- id- name@@ -16777,7 +19464,7 @@ components:mapping:type: arrayitems:- $ref: '#/components/schemas/unstableEvaluationRuleReadMapping'+ $ref: '#/components/schemas/unstablePromptVariableMappingRead'nullable: truedescription: >-Rule-specific override, or `null` to inherit the evaluator version's@@ -16788,13 +19475,14 @@ components:unstableLegacyEvaluationRuleEvaluatorAssignment:title: unstableLegacyEvaluationRuleEvaluatorAssignmenttype: object+ description: '**Deprecated:** Evaluator assignment on a legacy trace or dataset rule.'properties:evaluator:$ref: '#/components/schemas/unstableEvaluationRuleEvaluator'mapping:type: arrayitems:- $ref: '#/components/schemas/unstableLegacyEvaluationRuleMapping'+ $ref: '#/components/schemas/unstableLegacyPromptVariableMapping'nullable: truerequired:- evaluator@@ -16811,12 +19499,13 @@ components:enum:- trace- dataset- unstableLegacyEvaluationRuleMapping:- title: unstableLegacyEvaluationRuleMapping+ description: '**Deprecated:** Legacy trace or dataset evaluation target.'+ unstableLegacyPromptVariableMapping:+ title: unstableLegacyPromptVariableMappingtype: objectdescription: >-- Maps one evaluator variable to a trace, dataset item, or field on a- named observation in a legacy rule.+ **Deprecated:** Connects one prompt variable to data from a trace,+ dataset item, or named observation in a legacy rule.properties:variable:type: string@@ -16854,6 +19543,9 @@ components:- embedding- guardrail- dataset_item+ description: >-+ **Deprecated:** Legacy Langfuse object used by a trace or dataset+ evaluation rule.unstableCreateEvaluationRuleEvaluatorAssignment:title: unstableCreateEvaluationRuleEvaluatorAssignmenttype: object@@ -16863,7 +19555,7 @@ components:mapping:type: arrayitems:- $ref: '#/components/schemas/unstableEvaluationRuleMapping'+ $ref: '#/components/schemas/unstablePromptVariableMappingInput'nullable: truedescription: >-Optional rule-specific override. Omit to inherit the evaluator@@ -16977,7 +19669,7 @@ components:mapping:type: arrayitems:- $ref: '#/components/schemas/unstableEvaluationRuleReadMapping'+ $ref: '#/components/schemas/unstablePromptVariableMappingRead'nullable: truedescription: >-Default variable mapping for this evaluator version, or `null` when@@ -17136,7 +19828,7 @@ components:mapping:type: arrayitems:- $ref: '#/components/schemas/unstableEvaluationRuleMapping'+ $ref: '#/components/schemas/unstablePromptVariableMappingInput'nullable: truedescription: >-Optional default variable mapping inherited by rule assignments thatdiff --git a/web/src/__tests__/server/evaluation-rule-v2-service.servertest.ts b/web/src/__tests__/server/evaluation-rule-v2-service.servertest.tsindex 0644077bb5..11961f013c 100644--- a/web/src/__tests__/server/evaluation-rule-v2-service.servertest.ts+++ b/web/src/__tests__/server/evaluation-rule-v2-service.servertest.ts@@ -30,6 +30,7 @@ vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({const orgIds: string[] = [];let projectId = "";let otherProjectId = "";+let defaultEvaluatorId = "";beforeAll(async () => {const [first, second] = await Promise.all([@@ -41,9 +42,10 @@ beforeAll(async () => {orgIds.push(first.org.id, second.org.id);});-beforeEach(() => {+beforeEach(async () => {telemetryMocks.getRecentRuleExecutionTraces.mockReset();telemetryMocks.getTotalCostByRule.mockReset();+ defaultEvaluatorId = (await createEvaluator()).id;});afterEach(async () => {@@ -115,7 +117,7 @@ function createLegacyRule(});}-function createInput(evaluatorId?: string) {+function createInput(evaluatorId: string | null = defaultEvaluatorId) {return {projectId,name: "Production generations",@@ -416,17 +418,12 @@ describe("RuleService", () => {});describe("create", () => {- it("creates an active observation rule without evaluator assignments", async () => {+ it("rejects an active observation rule without evaluator assignments", async () => {await expect(- createService().create(createInput(), null),- ).resolves.toMatchObject({- enabled: true,- targetObject: "event",- timeScope: ["NEW"],- delay: 0,- sampling: 0.25,- assignments: [],- });+ createService().create(createInput(null), null),+ ).rejects.toThrow(+ "An enabled evaluation rule requires at least one evaluator assignment",+ );});it("returns evaluator mapping fallback data", async () => {@@ -721,7 +718,7 @@ describe("RuleService", () => {const evaluator = await createEvaluator();const service = createService();const rule = await service.create(- { ...createInput(), enabled: false },+ { ...createInput(null), enabled: false },null,);const attachment = {@@ -1055,7 +1052,17 @@ describe("RuleService", () => {});await prisma.evaluationRule.update({where: { id: legacyRule.id },- data: { status: "ACTIVE" },+ data: {+ status: "ACTIVE",+ filter: [+ {+ type: "string",+ column: "Trace Name",+ operator: "=",+ value: "legacy trace",+ },+ ],+ },});await service.update({projectId,@@ -1187,7 +1194,10 @@ describe("RuleService", () => {undefined,legacyMapping,);- const rule = await createService().create(createInput(), null);+ const rule = await createService().create(+ { ...createInput(null), enabled: false },+ null,+ );const updated = await createService().attach({projectId,@@ -1223,18 +1233,63 @@ describe("RuleService", () => {},]);});++ it("rejects enabling a rule whose resulting assignment list is empty", async () => {+ const service = createService();+ const evaluator = await createEvaluator();+ const assigned = await service.create(createInput(evaluator.id), null);+ const unassigned = await service.create(+ { ...createInput(null), enabled: false },+ null,+ );++ await expect(+ service.update({+ projectId,+ ruleId: assigned.id,+ enabled: true,+ evaluatorMappings: [],+ }),+ ).rejects.toThrow(+ "An enabled evaluation rule requires at least one evaluator assignment",+ );+ await expect(+ service.update({+ projectId,+ ruleId: unassigned.id,+ enabled: true,+ }),+ ).rejects.toThrow(+ "An enabled evaluation rule requires at least one evaluator assignment",+ );+ });});describe("setEnabled", () => {it("updates the rule status", async () => {+ const evaluator = await createEvaluator();const service = createService();- const rule = await service.create(createInput(), null);+ const rule = await service.create(createInput(evaluator.id), null);await expect(service.setEnabled({ projectId, ruleId: rule.id, enabled: false }),).resolves.toMatchObject({ id: rule.id, enabled: false });});+ it("rejects enabling a rule without evaluator assignments", async () => {+ const service = createService();+ const rule = await service.create(+ { ...createInput(null), enabled: false },+ null,+ );++ await expect(+ service.setEnabled({ projectId, ruleId: rule.id, enabled: true }),+ ).rejects.toThrow(+ "An enabled evaluation rule requires at least one evaluator assignment",+ );+ });+it("rejects an unavailable rule", async () => {await expect(createService().setEnabled({@@ -1325,6 +1380,31 @@ describe("RuleService", () => {});describe("setManyEnabled", () => {+ it("rejects enabling any selected rule without evaluator assignments", async () => {+ const service = createService();+ const [first, second] = await Promise.all([+ service.create({ ...createInput(null), enabled: false }, null),+ service.create(+ {+ ...createInput(null),+ name: "Second unassigned rule",+ enabled: false,+ },+ null,+ ),+ ]);++ await expect(+ service.setManyEnabled({+ projectId,+ ruleIds: [first.id, second.id],+ enabled: true,+ }),+ ).rejects.toThrow(+ "An enabled evaluation rule requires at least one evaluator assignment",+ );+ });+it("updates filtered matches without crossing project boundaries", async () => {const service = createService();const first = await service.create(createInput(), null);@@ -1332,9 +1412,10 @@ describe("RuleService", () => {{ ...createInput(), name: "Other production rule" },null,);+ const foreignEvaluator = await createEvaluator(otherProjectId);await createService().create({- ...createInput(),+ ...createInput(foreignEvaluator.id),projectId: otherProjectId,name: "Foreign production rule",},@@ -1345,13 +1426,24 @@ describe("RuleService", () => {projectId,isBatchAction: true,search: "production",+ filter: [+ {+ type: "string",+ column: "name",+ operator: "=",+ value: first.name,+ },+ ],enabled: false,});- expect(new Set(changedIds)).toEqual(new Set([first.id, second.id]));+ expect(changedIds).toEqual([first.id]);await expect(service.list({ projectId, page: 1, limit: 50, enabled: false }),- ).resolves.toMatchObject({ totalItems: 2 });+ ).resolves.toMatchObject({ totalItems: 1 });+ await expect(service.get(projectId, second.id)).resolves.toMatchObject({+ enabled: true,+ });});});@@ -1381,6 +1473,13 @@ describe("RuleService", () => {{ ...createInput(), name: "Second rule" },null,);+ await prisma.jobExecution.createMany({+ data: [first.id, second.id].map((jobConfigurationId) => ({+ projectId,+ jobConfigurationId,+ status: "PENDING" as const,+ })),+ });await expect(service.deleteMany({ projectId, ruleIds: [first.id, second.id] }),@@ -1388,14 +1487,23 @@ describe("RuleService", () => {await expect(service.list({ projectId, page: 1, limit: 50 }),).resolves.toEqual({ rules: [], totalItems: 0 });+ await expect(+ prisma.jobExecution.count({+ where: {+ projectId,+ jobConfigurationId: { in: [first.id, second.id] },+ },+ }),+ ).resolves.toBe(0);});it("ignores unavailable rules in an explicit selection", async () => {const service = createService();const local = await service.create(createInput(), null);+ const foreignEvaluator = await createEvaluator(otherProjectId);const foreign = await createService().create({- ...createInput(),+ ...createInput(foreignEvaluator.id),projectId: otherProjectId,name: "Foreign rule",},@@ -1456,10 +1564,13 @@ describe("RuleService", () => {},],});- await service.detach({+ await service.attach({projectId,ruleId: rule.id,- evaluatorId: secondEvaluator.id,+ assignment: {+ evaluatorId: firstEvaluator.id,+ variableMapping: null,+ },});await service.setManyEnabled({projectId,diff --git a/web/src/__tests__/server/evaluation-rule-v2-trpc.servertest.ts b/web/src/__tests__/server/evaluation-rule-v2-trpc.servertest.tsindex b2f6ac1c90..74b9a0a9cc 100644--- a/web/src/__tests__/server/evaluation-rule-v2-trpc.servertest.ts+++ b/web/src/__tests__/server/evaluation-rule-v2-trpc.servertest.ts@@ -125,7 +125,8 @@ function ruleInput(evaluatorId?: string) {describe("evaluation rule v2 tRPC", () => {it("supports the rule lifecycle", async () => {- const created = await caller.evalsV2.rules.create(ruleInput());+ const evaluator = await createEvaluator("Transport evaluator");+ const created = await caller.evalsV2.rules.create(ruleInput(evaluator.id));const updated = await caller.evalsV2.rules.update({projectId,ruleId: created.id,diff --git a/web/src/__tests__/server/evaluation-rules-public-api.servertest.ts b/web/src/__tests__/server/evaluation-rules-public-api.servertest.tsnew file mode 100644index 0000000000..4ccf22e21f--- /dev/null+++ b/web/src/__tests__/server/evaluation-rules-public-api.servertest.ts@@ -0,0 +1,613 @@+import { randomUUID } from "node:crypto";+import { EvalTargetObject, EvalTemplateType } from "@langfuse/shared";+import { prisma } from "@langfuse/shared/src/db";+import { createOrgProjectAndApiKey } from "@langfuse/shared/src/server";+import {+ makeAPICall,+ makeZodVerifiedAPICall,+} from "@/src/__tests__/test-utils";+import {+ createCodeEvaluator,+ createEvaluationRule,+} from "@/src/__tests__/server/stable-evaluation-public-api-test-utils";+import { EvaluatorService, RuleService } from "@/src/features/evals/server";+import {+ DeleteEvaluationRuleResponse,+ EvaluationRule,+ Evaluator,+ ListEvaluationRulesResponse,+ PublicApiError,+} from "@/src/features/public-api";++const createLegacyRule = (projectId: string, name: string) =>+ prisma.evaluationRule.create({+ data: {+ projectId,+ name,+ targetObject: EvalTargetObject.TRACE,+ status: "INACTIVE",+ filter: [+ {+ type: "string",+ column: "Trace Name",+ key: "legacy-filter-key",+ operator: "=",+ value: "legacy trace",+ },+ ],+ sampling: 1,+ delay: 0,+ timeScope: ["NEW"],+ },+ });++describe("stable evaluation rules public API", () => {+ it("creates evaluation rules", async () => {+ const { auth } = await createOrgProjectAndApiKey();+ const evaluator = await createCodeEvaluator("rule evaluator", auth);+ const datasetId = randomUUID();+ const body = {+ name: "same rule name",+ enabled: false,+ filter: [+ {+ type: "boolean" as const,+ column: "isExperimentItemRootSpan" as const,+ operator: "=" as const,+ value: true,+ },+ {+ type: "stringOptions" as const,+ column: "datasetId" as const,+ operator: "any of" as const,+ value: [datasetId],+ },+ {+ type: "null" as const,+ column: "parentObservationId" as const,+ operator: "is null" as const,+ value: "" as const,+ },+ ],+ evaluatorAssignments: [+ { evaluatorId: evaluator.body.id, variableMapping: null },+ ],+ };++ const first = await makeZodVerifiedAPICall(+ EvaluationRule,+ "POST",+ "/api/public/v2/evaluation-rules",+ body,+ auth,+ 201,+ );+ const second = await makeZodVerifiedAPICall(+ EvaluationRule,+ "POST",+ "/api/public/v2/evaluation-rules",+ body,+ auth,+ 201,+ );+ expect(first.body.id).not.toBe(second.body.id);+ expect(first.body).toMatchObject({+ createdBy: null,+ sampling: 1,+ evaluatorAssignments: [+ {+ evaluatorId: evaluator.body.id,+ variableMapping: null,+ },+ ],+ });+ expect(first.body).not.toHaveProperty("target");+ expect(first.body.filter).toEqual(+ expect.arrayContaining([+ expect.objectContaining({+ column: "isExperimentItemRootSpan",+ value: true,+ }),+ expect.objectContaining({+ column: "datasetId",+ value: [datasetId],+ }),+ expect.objectContaining({+ type: "null",+ column: "parentObservationId",+ operator: "is null",+ value: "",+ }),+ ]),+ );++ const nullSampling = await makeAPICall(+ "POST",+ "/api/public/v2/evaluation-rules",+ {+ ...body,+ name: "invalid nullable sampling",+ sampling: null,+ },+ auth,+ );+ expect(nullSampling.status).toBe(400);++ const enabledWithoutEvaluator = await makeAPICall(+ "POST",+ "/api/public/v2/evaluation-rules",+ {+ name: "invalid active draft",+ enabled: true,+ sampling: 1,+ filter: [],+ evaluatorAssignments: [],+ },+ auth,+ );+ expect(enabledWithoutEvaluator.status).toBe(400);+ });++ it("gets an evaluation rule", async () => {+ const { auth, projectId } = await createOrgProjectAndApiKey();+ const user = await prisma.user.create({+ data: {+ id: randomUUID(),+ name: "Rule Author",+ email: `${randomUUID()}@example.com`,+ },+ });+ const evaluator = await new EvaluatorService(+ prisma,+ async () => undefined,+ ).create(+ {+ projectId,+ name: "rule author evaluator",+ description: null,+ definition: {+ type: EvalTemplateType.CODE,+ sourceCode: "function evaluate() { return { scores: [] }; }",+ sourceCodeLanguage: "TYPESCRIPT",+ },+ },+ user.id,+ );+ const rule = await new RuleService(prisma, async () => undefined).create(+ {+ projectId,+ name: "authored rule",+ targetObject: EvalTargetObject.EVENT,+ enabled: false,+ sampling: 1,+ filter: [],+ evaluatorAssignments: [+ { evaluatorId: evaluator.id, variableMapping: null },+ ],+ },+ user.id,+ );++ const response = await makeZodVerifiedAPICall(+ EvaluationRule,+ "GET",+ `/api/public/v2/evaluation-rules/${rule.id}`,+ undefined,+ auth,+ );+ expect(response.body.createdBy).toEqual({+ id: user.id,+ name: user.name,+ });++ const legacy = await createLegacyRule(projectId, "legacy get rule");+ await prisma.evaluationRule.update({+ where: { id: legacy.id },+ data: {+ filter: [+ {+ type: "string",+ column: "Trace Name",+ operator: "=",+ value: "legacy trace",+ },+ {+ type: "booleanObject",+ column: "metadata",+ key: "reviewed",+ operator: "=",+ value: true,+ },+ {+ type: "positionInTrace",+ column: "positionInTrace",+ operator: "=",+ key: "root",+ },+ ],+ },+ });+ await prisma.evaluationRuleEvaluatorAssignment.create({+ data: {+ projectId,+ evaluationRuleId: legacy.id,+ evaluatorId: evaluator.id,+ variableMapping: [+ {+ templateVariable: "input",+ langfuseObject: "trace",+ objectName: null,+ selectedColumnId: "input",+ jsonSelector: null,+ },+ {+ templateVariable: "output",+ langfuseObject: "generation",+ objectName: "answer-generation",+ selectedColumnId: "output",+ jsonSelector: "$.answer",+ },+ ],+ },+ });+ const legacyResponse = await makeZodVerifiedAPICall(+ EvaluationRule,+ "GET",+ `/api/public/v2/evaluation-rules/${legacy.id}`,+ undefined,+ auth,+ );+ expect(legacyResponse.body).toMatchObject({+ id: legacy.id,+ filter: [+ { column: "Trace Name", value: "legacy trace" },+ {+ type: "booleanObject",+ column: "metadata",+ key: "reviewed",+ value: true,+ },+ {+ type: "positionInTrace",+ column: "positionInTrace",+ key: "root",+ },+ ],+ evaluatorAssignments: [+ {+ variableMapping: [+ {+ mappingType: "legacy",+ variable: "input",+ langfuseObject: "trace",+ objectName: null,+ source: "input",+ },+ {+ mappingType: "legacy",+ variable: "output",+ langfuseObject: "generation",+ objectName: "answer-generation",+ source: "output",+ jsonPath: "$.answer",+ },+ ],+ },+ ],+ });+ });++ it("returns 404 when getting an invalid evaluation rule ID", async () => {+ const { auth } = await createOrgProjectAndApiKey();+ const missing = await makeAPICall(+ "GET",+ `/api/public/v2/evaluation-rules/${randomUUID()}`,+ undefined,+ auth,+ );+ expect(missing.status).toBe(404);+ });++ it("rejects an LLM evaluator assignment when no variable mapping is configured", async () => {+ const { auth } = await createOrgProjectAndApiKey();+ const evaluator = await makeZodVerifiedAPICall(+ Evaluator,+ "POST",+ "/api/public/v2/evaluators",+ {+ name: "LLM evaluator without default mapping",+ type: "llm_as_judge",+ prompt: [{ role: "user", content: "Judge {{input}}" }],+ outputDefinition: {+ dataType: "NUMERIC",+ minValue: 0,+ maxValue: 1,+ },+ },+ auth,+ 201,+ );++ const rule = await makeAPICall(+ "POST",+ "/api/public/v2/evaluation-rules",+ {+ name: "Rule without variable mapping",+ enabled: false,+ evaluatorAssignments: [{ evaluatorId: evaluator.body.id }],+ },+ auth,+ );++ expect(rule.status).toBe(400);+ expect(PublicApiError.parse(rule.body)).toMatchObject({+ code: "invalid_request",+ message: "Missing mappings for evaluator variables: input",+ });+ });++ it("lists evaluation rules", async () => {+ const { auth, projectId } = await createOrgProjectAndApiKey();+ const evaluator = await createCodeEvaluator("listed rule evaluator", auth);+ const modern = await createEvaluationRule({+ name: "listed modern rule",+ evaluatorId: evaluator.body.id,+ auth,+ });+ const legacy = await createLegacyRule(projectId, "legacy listed rule");++ const response = await makeZodVerifiedAPICall(+ ListEvaluationRulesResponse,+ "GET",+ "/api/public/v2/evaluation-rules",+ undefined,+ auth,+ );+ expect(response.body.data).toContainEqual(+ expect.objectContaining({ id: modern.body.id }),+ );+ expect(response.body.data.find(({ id }) => id === legacy.id)).toMatchObject(+ {+ id: legacy.id,+ filter: [+ {+ type: "string",+ column: "Trace Name",+ key: "legacy-filter-key",+ operator: "=",+ value: "legacy trace",+ },+ ],+ },+ );+ });++ it("lists evaluation rules with cursor pagination", async () => {+ const { auth } = await createOrgProjectAndApiKey();+ const evaluator = await createCodeEvaluator(+ "paginated rule evaluator",+ auth,+ );+ const created = await Promise.all([+ createEvaluationRule({+ name: "paginated rule one",+ evaluatorId: evaluator.body.id,+ auth,+ }),+ createEvaluationRule({+ name: "paginated rule two",+ evaluatorId: evaluator.body.id,+ auth,+ }),+ ]);++ const firstPage = await makeZodVerifiedAPICall(+ ListEvaluationRulesResponse,+ "GET",+ "/api/public/v2/evaluation-rules?limit=1",+ undefined,+ auth,+ );+ expect(firstPage.body.data).toHaveLength(1);+ expect(firstPage.body.meta.cursor).toEqual(expect.any(String));++ const secondPage = await makeZodVerifiedAPICall(+ ListEvaluationRulesResponse,+ "GET",+ `/api/public/v2/evaluation-rules?limit=1&cursor=${encodeURIComponent(firstPage.body.meta.cursor!)}`,+ undefined,+ auth,+ );+ expect(secondPage.body.data).toHaveLength(1);+ expect(secondPage.body.meta.cursor).toBeUndefined();+ expect(+ new Set(+ [...firstPage.body.data, ...secondPage.body.data].map(({ id }) => id),+ ),+ ).toEqual(new Set(created.map(({ body: { id } }) => id)));+ });++ it("patches an evaluation rule", async () => {+ const { auth, projectId } = await createOrgProjectAndApiKey();+ const evaluator = await createCodeEvaluator("patched rule evaluator", auth);+ const rule = await createEvaluationRule({+ name: "rule before patch",+ evaluatorId: evaluator.body.id,+ auth,+ });++ const response = await makeZodVerifiedAPICall(+ EvaluationRule,+ "PATCH",+ `/api/public/v2/evaluation-rules/${rule.body.id}`,+ { name: "rule after patch", sampling: 0.5 },+ auth,+ );+ expect(response.body).toMatchObject({+ id: rule.body.id,+ name: "rule after patch",+ sampling: 0.5,+ });++ const contradictory = await makeAPICall(+ "PATCH",+ `/api/public/v2/evaluation-rules/${rule.body.id}`,+ { enabled: true, evaluatorAssignments: [] },+ auth,+ );+ expect(contradictory.status).toBe(400);+ expect(PublicApiError.parse(contradictory.body)).toMatchObject({+ code: "invalid_request",+ message:+ "An enabled evaluation rule requires at least one evaluator assignment",+ });++ const detached = await makeZodVerifiedAPICall(+ EvaluationRule,+ "PATCH",+ `/api/public/v2/evaluation-rules/${rule.body.id}`,+ { evaluatorAssignments: [] },+ auth,+ );+ expect(detached.body).toMatchObject({+ enabled: false,+ evaluatorAssignments: [],+ });++ const enableWithoutAssignments = await makeAPICall(+ "PATCH",+ `/api/public/v2/evaluation-rules/${rule.body.id}`,+ { enabled: true },+ auth,+ );+ expect(enableWithoutAssignments.status).toBe(400);++ const storedExperimentRule = await prisma.evaluationRule.create({+ data: {+ projectId,+ name: "stored experiment rule",+ targetObject: EvalTargetObject.EVENT,+ status: "INACTIVE",+ filter: [+ {+ type: "boolean",+ column: "isExperimentItemRootSpan",+ operator: "=",+ value: true,+ },+ ],+ sampling: 1,+ delay: 0,+ timeScope: ["NEW"],+ },+ });+ const experimentResponse = await makeZodVerifiedAPICall(+ EvaluationRule,+ "PATCH",+ `/api/public/v2/evaluation-rules/${storedExperimentRule.id}`,+ { filter: [] },+ auth,+ );+ expect(experimentResponse.body.filter).toEqual([]);+ await expect(+ prisma.evaluationRule.findUnique({+ where: { id: storedExperimentRule.id },+ select: { targetObject: true, filter: true },+ }),+ ).resolves.toMatchObject({+ targetObject: EvalTargetObject.EVENT,+ filter: [],+ });+ });++ it("matches the existing legacy rule update restrictions", async () => {+ const { auth, projectId } = await createOrgProjectAndApiKey();+ const legacy = await createLegacyRule(projectId, "legacy patch rule");++ const missing = await makeAPICall(+ "PATCH",+ `/api/public/v2/evaluation-rules/${randomUUID()}`,+ { name: "missing rule" },+ auth,+ );+ expect(missing.status).toBe(404);++ const legacyResponse = await makeZodVerifiedAPICall(+ EvaluationRule,+ "PATCH",+ `/api/public/v2/evaluation-rules/${legacy.id}`,+ { enabled: false },+ auth,+ );+ expect(legacyResponse.body).toMatchObject({+ id: legacy.id,+ enabled: false,+ });++ const unsupportedUpdate = await makeAPICall(+ "PATCH",+ `/api/public/v2/evaluation-rules/${legacy.id}`,+ { name: "mutated legacy rule" },+ auth,+ );+ expect(unsupportedUpdate.status).toBe(400);+ await expect(+ prisma.evaluationRule.findUnique({ where: { id: legacy.id } }),+ ).resolves.toMatchObject({+ name: "legacy patch rule",+ status: "INACTIVE",+ targetObject: EvalTargetObject.TRACE,+ });+ });++ it("deletes an evaluation rule", async () => {+ const { auth } = await createOrgProjectAndApiKey();+ const evaluator = await createCodeEvaluator("deleted rule evaluator", auth);+ const rule = await createEvaluationRule({+ name: "deleted rule",+ evaluatorId: evaluator.body.id,+ auth,+ });++ const response = await makeZodVerifiedAPICall(+ DeleteEvaluationRuleResponse,+ "DELETE",+ `/api/public/v2/evaluation-rules/${rule.body.id}`,+ undefined,+ auth,+ );+ expect(response.body.id).toBe(rule.body.id);++ const get = await makeAPICall(+ "GET",+ `/api/public/v2/evaluation-rules/${rule.body.id}`,+ undefined,+ auth,+ );+ expect(get.status).toBe(404);+ });++ it("deletes legacy rules and returns 404 for missing rules", async () => {+ const { auth, projectId } = await createOrgProjectAndApiKey();+ const legacy = await createLegacyRule(projectId, "legacy delete rule");+ const missing = await makeAPICall(+ "DELETE",+ `/api/public/v2/evaluation-rules/${randomUUID()}`,+ undefined,+ auth,+ );+ expect(missing.status).toBe(404);++ const legacyResponse = await makeZodVerifiedAPICall(+ DeleteEvaluationRuleResponse,+ "DELETE",+ `/api/public/v2/evaluation-rules/${legacy.id}`,+ undefined,+ auth,+ );+ expect(legacyResponse.body.id).toBe(legacy.id);+ await expect(+ prisma.evaluationRule.findUnique({ where: { id: legacy.id } }),+ ).resolves.toBeNull();+ });+});diff --git a/web/src/__tests__/server/evaluator-history-public-api.servertest.ts b/web/src/__tests__/server/evaluator-history-public-api.servertest.tsnew file mode 100644index 0000000000..15f1f213cf--- /dev/null+++ b/web/src/__tests__/server/evaluator-history-public-api.servertest.ts@@ -0,0 +1,104 @@+import { randomUUID } from "node:crypto";+import { createOrgProjectAndApiKey } from "@langfuse/shared/src/server";+import {+ makeAPICall,+ makeZodVerifiedAPICall,+} from "@/src/__tests__/test-utils";+import {+ codeDefinition,+ createCodeEvaluator,+} from "@/src/__tests__/server/stable-evaluation-public-api-test-utils";+import {+ Evaluator,+ ListEvaluatorVersionsResponse,+} from "@/src/features/public-api";++describe("stable evaluator history public API", () => {+ it("lists evaluator version history", async () => {+ const { auth } = await createOrgProjectAndApiKey();+ const evaluator = await createCodeEvaluator("history evaluator", auth);+ const changedSource =+ "function evaluate() { return { scores: [{ name: 'ok', value: true }] }; }";+ await makeZodVerifiedAPICall(+ Evaluator,+ "PATCH",+ `/api/public/v2/evaluators/${evaluator.body.id}`,+ codeDefinition(changedSource),+ auth,+ );++ const response = await makeZodVerifiedAPICall(+ ListEvaluatorVersionsResponse,+ "GET",+ `/api/public/v2/evaluators/${evaluator.body.id}/versions`,+ undefined,+ auth,+ );+ expect(response.body.data).toHaveLength(2);+ expect(response.body.data.map(({ version }) => version)).toEqual([2, 1]);+ expect(response.body.data[0]).toMatchObject({+ type: "code",+ version: 2,+ sourceCode: changedSource,+ createdBy: null,+ });+ expect(response.body.meta.cursor).toBeUndefined();+ });++ it("returns 404 for an invalid evaluator ID with or without a cursor", async () => {+ const { auth } = await createOrgProjectAndApiKey();+ const evaluatorId = randomUUID();+ const cursor = Buffer.from(JSON.stringify({ v: 1, version: 1 })).toString(+ "base64url",+ );++ for (const query of ["", `?cursor=${encodeURIComponent(cursor)}`]) {+ const response = await makeAPICall(+ "GET",+ `/api/public/v2/evaluators/${evaluatorId}/versions${query}`,+ undefined,+ auth,+ );+ expect(response.status).toBe(404);+ }+ });++ it("lists evaluator version history with cursor pagination", async () => {+ const { auth } = await createOrgProjectAndApiKey();+ const evaluator = await createCodeEvaluator(+ "paginated history evaluator",+ auth,+ );+ await makeZodVerifiedAPICall(+ Evaluator,+ "PATCH",+ `/api/public/v2/evaluators/${evaluator.body.id}`,+ codeDefinition(+ "function evaluate() { return { scores: [{ name: 'ok', value: true }] }; }",+ ),+ auth,+ );++ const firstPage = await makeZodVerifiedAPICall(+ ListEvaluatorVersionsResponse,+ "GET",+ `/api/public/v2/evaluators/${evaluator.body.id}/versions?limit=1`,+ undefined,+ auth,+ );+ expect(firstPage.body.data).toHaveLength(1);+ expect(firstPage.body.data[0]?.version).toBe(2);+ expect(firstPage.body.meta.cursor).toEqual(expect.any(String));++ const secondPage = await makeZodVerifiedAPICall(+ ListEvaluatorVersionsResponse,+ "GET",+ `/api/public/v2/evaluators/${evaluator.body.id}/versions?limit=1&cursor=${encodeURIComponent(firstPage.body.meta.cursor!)}`,+ undefined,+ auth,+ );+ expect(secondPage.body.data).toHaveLength(1);+ expect(secondPage.body.data[0]?.version).toBe(1);+ expect(secondPage.body.meta.cursor).toBeUndefined();+ });+});diff --git a/web/src/__tests__/server/evaluator-v2-repository.servertest.ts b/web/src/__tests__/server/evaluator-v2-repository.servertest.tsindex 1f5ef1b294..d9a0255a1f 100644--- a/web/src/__tests__/server/evaluator-v2-repository.servertest.ts+++ b/web/src/__tests__/server/evaluator-v2-repository.servertest.ts@@ -38,7 +38,6 @@ const llmDefinition = (vars: ["output"],variableMapping: null,outputDefinition: {- version: 2,dataType: "NUMERIC",score: { description: "Quality" },reasoning: { description: "Reasoning" },diff --git a/web/src/__tests__/server/evaluator-v2-service.servertest.ts b/web/src/__tests__/server/evaluator-v2-service.servertest.tsindex 3715e25e64..f8d32f5275 100644--- a/web/src/__tests__/server/evaluator-v2-service.servertest.ts+++ b/web/src/__tests__/server/evaluator-v2-service.servertest.ts@@ -37,6 +37,7 @@ import {const mocks = vi.hoisted(() => ({generateLangfuseAIText: vi.fn(),getRecentEvaluatorExecutionTraces: vi.fn(),+ invalidateProjectEvalConfigCaches: vi.fn(),assertEvaluatorConfigurationValid: vi.fn(),getEvaluatorDefinitionPreflightError: vi.fn(),testEvaluator: vi.fn(),@@ -53,6 +54,7 @@ vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({...(await importOriginal<typeof SharedServer>()),generateLangfuseAIText: mocks.generateLangfuseAIText,getRecentEvaluatorExecutionTraces: mocks.getRecentEvaluatorExecutionTraces,+ invalidateProjectEvalConfigCaches: mocks.invalidateProjectEvalConfigCaches,}));vi.mock("@/src/env.mjs", async (importOriginal) => {@@ -114,7 +116,6 @@ const llmInput = ({ templateVariable: "output", selectedColumnId: "output" },],outputDefinition: {- version: 2,dataType: "NUMERIC",score: { description: "Quality" },reasoning: { description: "Reasoning" },@@ -163,6 +164,7 @@ beforeAll(async () => {beforeEach(() => {mocks.generateLangfuseAIText.mockReset();mocks.getRecentEvaluatorExecutionTraces.mockReset();+ mocks.invalidateProjectEvalConfigCaches.mockReset();mocks.assertEvaluatorConfigurationValid.mockReset();mocks.assertEvaluatorConfigurationValid.mockResolvedValue(undefined);mocks.getEvaluatorDefinitionPreflightError.mockReset();@@ -490,6 +492,84 @@ describe("EvaluatorService", () => {"Evaluator paused: no valid evaluation model is configured. Update the evaluator template or default evaluation model, then reactivate it.",versions: [expect.objectContaining({ version: 2 })],});++ mocks.invalidateProjectEvalConfigCaches.mockClear();+ const recovered = await service.patch(+ {+ projectId,+ evaluatorId: evaluator.id,+ definition: {+ ...llmInput("Invalid model").definition,+ provider: "openai",+ model: "available-model",+ },+ },+ null,+ );+ expect(recovered).toMatchObject({+ id: evaluator.id,+ blockedAt: null,+ blockReason: null,+ blockMessage: null,+ versions: [expect.objectContaining({ version: 3 })],+ });+ expect(mocks.invalidateProjectEvalConfigCaches).toHaveBeenCalledWith(+ projectId,+ );++ await prisma.evaluator.update({+ where: { id: evaluator.id },+ data: {+ blockedAt: new Date(),+ blockReason: "LLM_CONNECTION_AUTH_INVALID",+ blockMessage: "Authentication failed",+ },+ });+ const stillBlocked = await service.patch(+ {+ projectId,+ evaluatorId: evaluator.id,+ definition: {+ ...llmInput("Invalid model").definition,+ prompt: "Updated prompt: {{output}}",+ provider: "openai",+ model: "available-model",+ },+ },+ null,+ );+ expect(stillBlocked).toMatchObject({+ blockedAt: expect.any(Date),+ blockReason: "LLM_CONNECTION_AUTH_INVALID",+ blockMessage: "Authentication failed",+ });++ await prisma.evaluator.update({+ where: { id: evaluator.id },+ data: {+ blockedAt: new Date(),+ blockReason: null,+ blockMessage: "Legacy pause without a reason",+ },+ });+ const legacyPause = await service.patch(+ {+ projectId,+ evaluatorId: evaluator.id,+ definition: {+ ...llmInput("Invalid model").definition,+ prompt: "Another updated prompt: {{output}}",+ provider: "openai",+ model: "available-model",+ },+ },+ null,+ );+ expect(legacyPause).toMatchObject({+ blockedAt: expect.any(Date),+ blockReason: null,+ blockMessage: "Legacy pause without a reason",+ });});it("reactivates a blocked evaluator only after its model test succeeds", async () => {diff --git a/web/src/__tests__/server/evaluator-v2-trpc.servertest.ts b/web/src/__tests__/server/evaluator-v2-trpc.servertest.tsindex 961e716d9a..9d85dfb987 100644--- a/web/src/__tests__/server/evaluator-v2-trpc.servertest.ts+++ b/web/src/__tests__/server/evaluator-v2-trpc.servertest.ts@@ -34,13 +34,9 @@ let session: Session & { user: NonNullable<Session["user"]> };const definition = {type: "LLM_AS_JUDGE" as const,prompt: "Judge {{output}}",- provider: null,- model: null,- modelParams: null,- vars: ["output"],+ modelConfig: null,variableMapping: [{ templateVariable: "output", selectedColumnId: "output" }],outputDefinition: {- version: 2 as const,dataType: "NUMERIC" as const,score: { description: "Quality" },reasoning: { description: "Reasoning" },diff --git a/web/src/__tests__/server/evaluators-public-api.servertest.ts b/web/src/__tests__/server/evaluators-public-api.servertest.tsnew file mode 100644index 0000000000..ef27d3601c--- /dev/null+++ b/web/src/__tests__/server/evaluators-public-api.servertest.ts@@ -0,0 +1,663 @@+import { randomUUID } from "node:crypto";+import { EvalTargetObject, EvalTemplateType } from "@langfuse/shared";+import { prisma } from "@langfuse/shared/src/db";+import { createOrgProjectAndApiKey } from "@langfuse/shared/src/server";+import {+ makeAPICall,+ makeZodVerifiedAPICall,+} from "@/src/__tests__/test-utils";+import {+ codeDefinition,+ createCodeEvaluator,+ createEvaluationRule,+} from "@/src/__tests__/server/stable-evaluation-public-api-test-utils";+import { EvaluatorService, RuleService } from "@/src/features/evals/server";+import {+ DeleteEvaluatorResponse,+ Evaluator,+ ListEvaluatorsResponse,+ LlmAsJudgeEvaluator,+ PublicApiError,+} from "@/src/features/public-api";++describe("stable evaluators public API", () => {+ it("creates evaluators", async () => {+ const { auth } = await createOrgProjectAndApiKey();++ const first = await createCodeEvaluator("same name", auth);+ const second = await createCodeEvaluator("same name", auth);+ expect(first.body.id).not.toBe(second.body.id);+ expect(first.body).toMatchObject({+ name: "same name",+ type: "code",+ createdBy: null,+ version: 1,+ versionCreatedBy: null,+ });+ expect(first.body).not.toHaveProperty("latestVersion");++ const llmEvaluator = await makeZodVerifiedAPICall(+ Evaluator,+ "POST",+ "/api/public/v2/evaluators",+ {+ name: "friendly LLM evaluator",+ type: "llm_as_judge",+ prompt: "Input: {{input}}",+ modelConfig: null,+ variableMapping: null,+ outputDefinition: {+ dataType: "NUMERIC",+ minValue: 0,+ maxValue: 1,+ },+ },+ auth,+ 201,+ );+ const llmEvaluatorBody = LlmAsJudgeEvaluator.parse(llmEvaluator.body);+ expect(llmEvaluatorBody).toMatchObject({+ type: "llm_as_judge",+ version: 1,+ prompt: [{ role: "user", content: "Input: {{input}}" }],+ variables: ["input"],+ variableMapping: null,+ modelConfig: null,+ });+ expect(llmEvaluatorBody.outputDefinition).toEqual({+ dataType: "NUMERIC",+ minValue: 0,+ maxValue: 1,+ });+ await expect(+ prisma.evaluatorVersion.findFirstOrThrow({+ where: { evaluatorId: llmEvaluator.body.id },+ orderBy: { version: "desc" },+ select: { prompt: true },+ }),+ ).resolves.toEqual({ prompt: "Input: {{input}}" });++ const categoricalEvaluator = await makeZodVerifiedAPICall(+ Evaluator,+ "POST",+ "/api/public/v2/evaluators",+ {+ name: "categorical evaluator",+ type: "llm_as_judge",+ prompt: [{ role: "user", content: "Classify: {{input}}" }],+ outputDefinition: {+ dataType: "CATEGORICAL",+ scoreReasoningInstructions: "Why the category was selected",+ scoreValueInstructions: "The selected category",+ categories: ["pass", "fail"],+ shouldAllowMultipleMatches: false,+ },+ },+ auth,+ 201,+ );+ expect(+ LlmAsJudgeEvaluator.parse(categoricalEvaluator.body).outputDefinition,+ ).toEqual({+ dataType: "CATEGORICAL",+ scoreReasoningInstructions: "Why the category was selected",+ scoreValueInstructions: "The selected category",+ categories: ["pass", "fail"],+ shouldAllowMultipleMatches: false,+ });++ const patchedEvaluator = await makeZodVerifiedAPICall(+ Evaluator,+ "PATCH",+ `/api/public/v2/evaluators/${categoricalEvaluator.body.id}`,+ {+ type: "llm_as_judge",+ prompt: "Classify updated: {{input}}",+ outputDefinition: {+ dataType: "BOOLEAN",+ },+ },+ auth,+ );+ expect(LlmAsJudgeEvaluator.parse(patchedEvaluator.body).prompt).toEqual([+ { role: "user", content: "Classify updated: {{input}}" },+ ]);++ const tooManyMessages = await makeAPICall(+ "POST",+ "/api/public/v2/evaluators",+ {+ name: "too many messages",+ type: "llm_as_judge",+ prompt: [+ { role: "user", content: "First" },+ { role: "user", content: "Second" },+ ],+ outputDefinition: { dataType: "BOOLEAN" },+ },+ auth,+ );+ expect(tooManyMessages.status).toBe(400);++ for (const role of ["assistant", "system", "developer"]) {+ const invalidRole = await makeAPICall(+ "POST",+ "/api/public/v2/evaluators",+ {+ name: "invalid role evaluator",+ type: "llm_as_judge",+ prompt: [{ role, content: "Judge {{input}}" }],+ outputDefinition: { dataType: "BOOLEAN" },+ },+ auth,+ );+ expect(invalidRole.status).toBe(400);+ expect(PublicApiError.parse(invalidRole.body)).toMatchObject({+ code: "invalid_body",+ });+ }++ const invalidModelParams = await makeAPICall(+ "POST",+ "/api/public/v2/evaluators",+ {+ name: "invalid model params",+ type: "llm_as_judge",+ prompt: "Judge {{input}}",+ modelConfig: {+ provider: "openai",+ model: "gpt-4.1-mini",+ modelParams: { temperature: 0 },+ },+ outputDefinition: { dataType: "BOOLEAN" },+ },+ auth,+ );+ expect(invalidModelParams.status).toBe(400);++ const invalid = await makeAPICall(+ "POST",+ "/api/public/v2/evaluators",+ { name: "missing type" },+ auth,+ );+ expect(invalid.status).toBe(400);+ expect(PublicApiError.parse(invalid.body)).toMatchObject({+ code: "invalid_body",+ details: { issues: expect.any(Array) },+ });++ const nestedDefinition = await makeAPICall(+ "POST",+ "/api/public/v2/evaluators",+ {+ name: "nested definition",+ definition: codeDefinition("function evaluate() {}"),+ },+ auth,+ );+ expect(nestedDefinition.status).toBe(400);+ });++ it("gets an evaluator", async () => {+ const { auth, projectId } = await createOrgProjectAndApiKey();+ const user = await prisma.user.create({+ data: {+ id: randomUUID(),+ name: "Evaluator Author",+ email: `${randomUUID()}@example.com`,+ },+ });+ const evaluator = await new EvaluatorService(+ prisma,+ async () => undefined,+ ).create(+ {+ projectId,+ name: "authored evaluator",+ description: null,+ definition: {+ type: EvalTemplateType.LLM_AS_JUDGE,+ prompt: "Judge {{input}}",+ vars: ["input"],+ provider: null,+ model: null,+ modelParams: null,+ variableMapping: null,+ outputDefinition: {+ dataType: "NUMERIC",+ reasoning: { description: "Explain the score" },+ score: { description: "Return the score" },+ },+ },+ },+ user.id,+ );+ const rule = await new RuleService(prisma, async () => undefined).create(+ {+ projectId,+ name: "associated rule",+ targetObject: EvalTargetObject.EVENT,+ enabled: false,+ sampling: 1,+ filter: [],+ evaluatorAssignments: [+ {+ evaluatorId: evaluator.id,+ variableMapping: [+ { templateVariable: "input", selectedColumnId: "input" },+ ],+ },+ ],+ },+ user.id,+ );++ const legacyRule = await prisma.evaluationRule.create({+ data: {+ projectId,+ name: "legacy associated rule",+ targetObject: EvalTargetObject.TRACE,+ status: "INACTIVE",+ filter: [],+ sampling: 1,+ delay: 0,+ timeScope: ["NEW"],+ },+ });+ await prisma.evaluationRuleEvaluatorAssignment.create({+ data: {+ projectId,+ evaluationRuleId: legacyRule.id,+ evaluatorId: evaluator.id,+ variableMapping: [+ {+ templateVariable: "input",+ langfuseObject: "trace",+ objectName: null,+ selectedColumnId: "input",+ jsonSelector: null,+ },+ ],+ },+ });++ const response = await makeZodVerifiedAPICall(+ Evaluator,+ "GET",+ `/api/public/v2/evaluators/${evaluator.id}`,+ undefined,+ auth,+ );+ expect(response.body.createdBy).toEqual({+ id: user.id,+ name: user.name,+ });+ expect(response.body.versionCreatedBy).toEqual(response.body.createdBy);+ const expectedAssignments = expect.arrayContaining([+ {+ evaluationRuleId: rule.id,+ variableMappingOverride: [{ variable: "input", source: "input" }],+ },+ {+ evaluationRuleId: legacyRule.id,+ variableMappingOverride: [+ {+ mappingType: "legacy",+ variable: "input",+ langfuseObject: "trace",+ objectName: null,+ source: "input",+ },+ ],+ },+ ]);+ expect(response.body.evaluationRuleAssignments).toEqual(+ expectedAssignments,+ );++ const listResponse = await makeZodVerifiedAPICall(+ ListEvaluatorsResponse,+ "GET",+ "/api/public/v2/evaluators",+ undefined,+ auth,+ );+ expect(+ listResponse.body.data.find(({ id }) => id === evaluator.id)+ ?.evaluationRuleAssignments,+ ).toEqual(expectedAssignments);++ const legacyOutputEvaluator = await prisma.evaluator.create({+ data: {+ projectId,+ name: "legacy output evaluator",+ description: null,+ type: EvalTemplateType.LLM_AS_JUDGE,+ versions: {+ create: {+ version: 1,+ prompt: "Judge {{input}}",+ vars: ["input"],+ outputDefinition: { reasoning: "", score: "" },+ },+ },+ },+ });+ const legacyResponse = await makeZodVerifiedAPICall(+ Evaluator,+ "GET",+ `/api/public/v2/evaluators/${legacyOutputEvaluator.id}`,+ undefined,+ auth,+ );+ expect(legacyResponse.body).toMatchObject({+ version: 1,+ prompt: [{ role: "user", content: "Judge {{input}}" }],+ outputDefinition: {+ dataType: "NUMERIC",+ },+ });+ });++ it("returns 404 when getting an invalid evaluator ID", async () => {+ const { auth } = await createOrgProjectAndApiKey();++ const response = await makeAPICall(+ "GET",+ `/api/public/v2/evaluators/${randomUUID()}`,+ undefined,+ auth,+ );+ expect(response.status).toBe(404);+ expect(PublicApiError.parse(response.body)).toMatchObject({+ code: "resource_not_found",+ message: "Evaluator not found",+ });+ });++ it("lists evaluators", async () => {+ const { auth } = await createOrgProjectAndApiKey();+ const created = await createCodeEvaluator("listed evaluator", auth);++ const response = await makeZodVerifiedAPICall(+ ListEvaluatorsResponse,+ "GET",+ "/api/public/v2/evaluators",+ undefined,+ auth,+ );+ expect(response.body.data).toContainEqual(+ expect.objectContaining({+ id: created.body.id,+ name: "listed evaluator",+ }),+ );++ const invalidQuery = await makeAPICall(+ "GET",+ "/api/public/v2/evaluators?limit=101",+ undefined,+ auth,+ );+ expect(invalidQuery.status).toBe(400);+ expect(PublicApiError.parse(invalidQuery.body)).toMatchObject({+ code: "invalid_query",+ details: { issues: expect.any(Array) },+ });+ });++ it("lists evaluators with cursor pagination", async () => {+ const { auth } = await createOrgProjectAndApiKey();+ const created = await Promise.all([+ createCodeEvaluator("cursor one", auth),+ createCodeEvaluator("cursor two", auth),+ createCodeEvaluator("cursor three", auth),+ ]);++ const firstPage = await makeZodVerifiedAPICall(+ ListEvaluatorsResponse,+ "GET",+ "/api/public/v2/evaluators?limit=2",+ undefined,+ auth,+ );+ expect(firstPage.body.data).toHaveLength(2);+ expect(firstPage.body.meta.cursor).toEqual(expect.any(String));++ const secondPage = await makeZodVerifiedAPICall(+ ListEvaluatorsResponse,+ "GET",+ `/api/public/v2/evaluators?limit=2&cursor=${encodeURIComponent(firstPage.body.meta.cursor!)}`,+ undefined,+ auth,+ );+ expect(secondPage.body.data).toHaveLength(1);+ expect(secondPage.body.meta.cursor).toBeUndefined();++ const listedIds = [...firstPage.body.data, ...secondPage.body.data].map(+ ({ id }) => id,+ );+ expect(new Set(listedIds)).toEqual(+ new Set(created.map(({ body }) => body.id)),+ );+ });++ it("preserves model configuration when patching another field", async () => {+ const { auth, projectId } = await createOrgProjectAndApiKey();+ const evaluator = await new EvaluatorService(+ prisma,+ async () => undefined,+ ).create(+ {+ projectId,+ name: "configured evaluator",+ description: null,+ definition: {+ type: EvalTemplateType.LLM_AS_JUDGE,+ prompt: "Judge {{input}}",+ vars: ["input"],+ provider: "openai",+ model: "gpt-4.1-mini",+ modelParams: { temperature: 0.2 },+ variableMapping: null,+ outputDefinition: {+ dataType: "BOOLEAN",+ reasoning: { description: "Explain the score" },+ score: { description: "Return the score" },+ },+ },+ },+ null,+ );++ await makeZodVerifiedAPICall(+ Evaluator,+ "PATCH",+ `/api/public/v2/evaluators/${evaluator.id}`,+ {+ type: "llm_as_judge",+ prompt: "Judge carefully: {{input}}",+ outputDefinition: { dataType: "BOOLEAN" },+ },+ auth,+ );++ await expect(+ prisma.evaluatorVersion.findFirstOrThrow({+ where: { evaluatorId: evaluator.id },+ orderBy: { version: "desc" },+ select: {+ provider: true,+ model: true,+ modelParams: true,+ },+ }),+ ).resolves.toEqual({+ provider: "openai",+ model: "gpt-4.1-mini",+ modelParams: { temperature: 0.2 },+ });+ });++ it("patches an evaluator", async () => {+ const { auth } = await createOrgProjectAndApiKey();+ const evaluator = await createCodeEvaluator("concurrent evaluator", auth);+ const rule = await createEvaluationRule({+ name: "recovery assignment",+ evaluatorId: evaluator.body.id,+ auth,+ });+ await prisma.evaluator.update({+ where: { id: evaluator.body.id },+ data: {+ blockedAt: new Date(),+ blockReason: "EVAL_MODEL_CONFIG_INVALID",+ blockMessage: "Invalid model configuration",+ },+ });++ const recovered = await makeZodVerifiedAPICall(+ Evaluator,+ "PATCH",+ `/api/public/v2/evaluators/${evaluator.body.id}`,+ codeDefinition("function evaluate() { return { scores: [] }; }"),+ auth,+ );+ expect(recovered.body).toMatchObject({+ id: evaluator.body.id,+ status: "active",+ pausedAt: null,+ pausedReason: null,+ pausedMessage: null,+ version: 1,+ evaluationRuleAssignments: [{ evaluationRuleId: rule.body.id }],+ });++ const metadataUpdate = await makeZodVerifiedAPICall(+ Evaluator,+ "PATCH",+ `/api/public/v2/evaluators/${evaluator.body.id}`,+ { description: "metadata-only description" },+ auth,+ );+ expect(metadataUpdate.body.version).toBe(1);++ const partialDefinition = await makeAPICall(+ "PATCH",+ `/api/public/v2/evaluators/${evaluator.body.id}`,+ { type: "code", sourceCode: "function evaluate() {}" },+ auth,+ );+ expect(partialDefinition.status).toBe(400);+ expect(PublicApiError.parse(partialDefinition.body).code).toBe(+ "invalid_body",+ );++ const changedSource =+ "function evaluate() { return { scores: [{ name: 'ok', value: true }] }; }";+ await Promise.all([+ makeZodVerifiedAPICall(+ Evaluator,+ "PATCH",+ `/api/public/v2/evaluators/${evaluator.body.id}`,+ { name: "concurrent renamed" },+ auth,+ ),+ makeZodVerifiedAPICall(+ Evaluator,+ "PATCH",+ `/api/public/v2/evaluators/${evaluator.body.id}`,+ { description: "concurrent description" },+ auth,+ ),+ makeZodVerifiedAPICall(+ Evaluator,+ "PATCH",+ `/api/public/v2/evaluators/${evaluator.body.id}`,+ codeDefinition(changedSource),+ auth,+ ),+ ]);++ const current = await makeZodVerifiedAPICall(+ Evaluator,+ "GET",+ `/api/public/v2/evaluators/${evaluator.body.id}`,+ undefined,+ auth,+ );+ expect(current.body).toMatchObject({+ name: "concurrent renamed",+ description: "concurrent description",+ version: 2,+ sourceCode: changedSource,+ });++ const typeConflict = await makeAPICall(+ "PATCH",+ `/api/public/v2/evaluators/${evaluator.body.id}`,+ {+ type: "llm_as_judge",+ prompt: "Input: {{input}}",+ modelConfig: null,+ variableMapping: null,+ outputDefinition: {+ dataType: "BOOLEAN",+ },+ },+ auth,+ );+ expect(typeConflict.status).toBe(409);+ expect(PublicApiError.parse(typeConflict.body)).toMatchObject({+ code: "conflict",+ message: "Evaluator type cannot be changed",+ });+ });++ it("returns 404 when patching an invalid evaluator ID", async () => {+ const { auth } = await createOrgProjectAndApiKey();++ const response = await makeAPICall(+ "PATCH",+ `/api/public/v2/evaluators/${randomUUID()}`,+ { name: "missing evaluator" },+ auth,+ );+ expect(response.status).toBe(404);+ });++ it("deletes an evaluator", async () => {+ const { auth } = await createOrgProjectAndApiKey();+ const evaluator = await createCodeEvaluator("deleted evaluator", auth);++ const response = await makeZodVerifiedAPICall(+ DeleteEvaluatorResponse,+ "DELETE",+ `/api/public/v2/evaluators/${evaluator.body.id}`,+ undefined,+ auth,+ );+ expect(response.body.id).toBe(evaluator.body.id);++ const get = await makeAPICall(+ "GET",+ `/api/public/v2/evaluators/${evaluator.body.id}`,+ undefined,+ auth,+ );+ expect(get.status).toBe(404);+ });++ it("returns 404 when deleting an invalid evaluator ID", async () => {+ const { auth } = await createOrgProjectAndApiKey();++ const response = await makeAPICall(+ "DELETE",+ `/api/public/v2/evaluators/${randomUUID()}`,+ undefined,+ auth,+ );+ expect(response.status).toBe(404);+ });+});diff --git a/web/src/__tests__/server/stable-evaluation-public-api-test-utils.ts b/web/src/__tests__/server/stable-evaluation-public-api-test-utils.tsnew file mode 100644index 0000000000..d6345f7d77--- /dev/null+++ b/web/src/__tests__/server/stable-evaluation-public-api-test-utils.ts@@ -0,0 +1,43 @@+import { makeZodVerifiedAPICall } from "@/src/__tests__/test-utils";+import { EvaluationRule, Evaluator } from "@/src/features/public-api";++export const codeDefinition = (sourceCode: string) => ({+ type: "code" as const,+ sourceCode,+ sourceCodeLanguage: "TYPESCRIPT" as const,+});++export const createCodeEvaluator = (name: string, auth: string) =>+ makeZodVerifiedAPICall(+ Evaluator,+ "POST",+ "/api/public/v2/evaluators",+ {+ name,+ description: `${name} description`,+ ...codeDefinition("function evaluate() { return { scores: [] }; }"),+ },+ auth,+ 201,+ );++export const createEvaluationRule = (params: {+ name: string;+ evaluatorId: string;+ auth: string;+}) =>+ makeZodVerifiedAPICall(+ EvaluationRule,+ "POST",+ "/api/public/v2/evaluation-rules",+ {+ name: params.name,+ enabled: false,+ filter: [],+ evaluatorAssignments: [+ { evaluatorId: params.evaluatorId, variableMapping: null },+ ],+ },+ params.auth,+ 201,+ );diff --git a/web/src/__tests__/server/unit/create-authed-project-api-route-auth-errors.servertest.ts b/web/src/__tests__/server/unit/create-authed-project-api-route-auth-errors.servertest.tsindex 901882a51d..ad7ee6b232 100644--- a/web/src/__tests__/server/unit/create-authed-project-api-route-auth-errors.servertest.ts+++ b/web/src/__tests__/server/unit/create-authed-project-api-route-auth-errors.servertest.ts@@ -8,16 +8,16 @@ const {mockRateLimitRequest,mockTraceException,mockLoggerDebug,- mockCreateUnstablePublicApiAuthError,- mockSendUnstablePublicApiErrorResponse,+ mockCreateStructuredPublicApiAuthError,+ mockSendStructuredPublicApiErrorResponse,} = vi.hoisted(() => ({mockVerifyAuthHeaderAndReturnScope: vi.fn(),mockIsPrismaException: vi.fn(),mockRateLimitRequest: vi.fn(),mockTraceException: vi.fn(),mockLoggerDebug: vi.fn(),- mockCreateUnstablePublicApiAuthError: vi.fn((value) => value),- mockSendUnstablePublicApiErrorResponse: vi.fn(),+ mockCreateStructuredPublicApiAuthError: vi.fn((value) => value),+ mockSendStructuredPublicApiErrorResponse: vi.fn(),}));vi.mock("@/src/features/public-api/server/apiAuth", () => ({@@ -60,12 +60,13 @@ vi.mock("@/src/features/public-api/server/withMiddlewares", () => ({}));vi.mock(- "@/src/features/public-api/server/unstable-public-api-error-contract",+ "@/src/features/public-api/server/structuredPublicApiErrorContract",() => ({- unstablePublicEvalsErrorContract: "unstable-public-evals",- createUnstablePublicApiAuthError: mockCreateUnstablePublicApiAuthError,- createUnstablePublicApiRequestValidationError: vi.fn(),- sendUnstablePublicApiErrorResponse: mockSendUnstablePublicApiErrorResponse,+ structuredPublicApiErrorContract: "structured",+ createStructuredPublicApiAuthError: mockCreateStructuredPublicApiAuthError,+ createStructuredPublicApiRequestValidationError: vi.fn(),+ sendStructuredPublicApiErrorResponse:+ mockSendStructuredPublicApiErrorResponse,}),);@@ -112,7 +113,7 @@ describe("createAuthedProjectAPIRoute auth error handling", () => {});async function callRoute(options?: {- useUnstableErrorContract?: boolean;+ useStructuredErrorContract?: boolean;rateLimitUpgradePath?: {legacyEndpoint: string;replacementEndpoint: string;@@ -124,8 +125,8 @@ describe("createAuthedProjectAPIRoute auth error handling", () => {name: "Test Route",querySchema: z.object({}),responseSchema: z.object({ ok: z.literal(true) }),- errorContract: options?.useUnstableErrorContract- ? "unstable-public-evals"+ errorContract: options?.useStructuredErrorContract+ ? "structured": undefined,rateLimitUpgradePath: options?.rateLimitUpgradePath,fn: async () => ({ ok: true as const }),@@ -183,19 +184,34 @@ describe("createAuthedProjectAPIRoute auth error handling", () => {expect(mockTraceException).toHaveBeenCalledWith(prismaLikeError);});- it("returns unstable auth errors and traces prisma auth failures", async () => {+ it("returns structured authentication errors", async () => {+ mockVerifyAuthHeaderAndReturnScope.mockResolvedValueOnce({+ validKey: false,+ error: "Invalid credentials",+ });++ await callRoute({ useStructuredErrorContract: true });++ expect(mockCreateStructuredPublicApiAuthError).toHaveBeenCalledWith({+ statusCode: 401,+ message: "Invalid credentials",+ });+ expect(mockSendStructuredPublicApiErrorResponse).toHaveBeenCalledTimes(1);+ });++ it("returns structured errors and traces prisma auth failures", async () => {const prismaLikeError = new Error("Can't reach database server");mockVerifyAuthHeaderAndReturnScope.mockRejectedValueOnce(prismaLikeError);mockIsPrismaException.mockReturnValue(true);- await callRoute({ useUnstableErrorContract: true });+ await callRoute({ useStructuredErrorContract: true });expect(mockTraceException).toHaveBeenCalledWith(prismaLikeError);- expect(mockCreateUnstablePublicApiAuthError).toHaveBeenCalledWith({+ expect(mockCreateStructuredPublicApiAuthError).toHaveBeenCalledWith({statusCode: 503,message: "Service Unavailable",});- expect(mockSendUnstablePublicApiErrorResponse).toHaveBeenCalledTimes(1);+ expect(mockSendStructuredPublicApiErrorResponse).toHaveBeenCalledTimes(1);});it("keeps the shared rate limit response for routes without upgrade guidance", async () => {diff --git a/web/src/__tests__/server/unit/dashboard-widget-min-version.servertest.ts b/web/src/__tests__/server/unit/dashboard-widget-min-version.servertest.tsindex 6a91a557aa..133b4a8633 100644--- a/web/src/__tests__/server/unit/dashboard-widget-min-version.servertest.ts+++ b/web/src/__tests__/server/unit/dashboard-widget-min-version.servertest.ts@@ -2,7 +2,7 @@ import {normalizePublicDashboardWidgetInput,validatePublicDashboardWidgetInput,} from "@/src/features/widgets/server/public-dashboard-widget-service";-import { UnstablePublicApiError } from "@/src/features/public-api/server/unstable-public-api-error-contract";+import { StructuredPublicApiError } from "@/src/features/public-api";import { env as sharedEnv } from "@langfuse/shared/src/env";const baseInput = {@@ -82,8 +82,8 @@ describe("public dashboard widget version validation", () => {validationError = error;}- expect(validationError).toBeInstanceOf(UnstablePublicApiError);- const allowedValues = (validationError as UnstablePublicApiError).details+ expect(validationError).toBeInstanceOf(StructuredPublicApiError);+ const allowedValues = (validationError as StructuredPublicApiError).details?.allowedValues;expect(allowedValues?.length).toBeGreaterThan(0);diff --git a/web/src/__tests__/server/unit/evaluator-input.servertest.ts b/web/src/__tests__/server/unit/evaluator-input.servertest.tsnew file mode 100644index 0000000000..a03a62897c--- /dev/null+++ b/web/src/__tests__/server/unit/evaluator-input.servertest.ts@@ -0,0 +1,95 @@+import { EvalTemplateType } from "@langfuse/shared";+import { describe, expect, it } from "vitest";+import { z } from "zod";+import {+ McpEvaluatorInput,+ McpEvaluatorInputBase,+} from "@/src/features/mcp/features/evals/tools/evaluatorInput";++const llmEvaluatorInput = {+ name: "Answer quality",+ type: EvalTemplateType.LLM_AS_JUDGE,+ prompt: "Judge {{output}}",+ outputDefinition: {+ dataType: "NUMERIC",+ reasoning: { description: "Explain the score" },+ score: { description: "Return the score" },+ },+};++const hasJsonSchemaComposition = (value: unknown): boolean => {+ if (typeof value !== "object" || value === null) return false;+ if (Array.isArray(value)) return value.some(hasJsonSchemaComposition);++ const schema = value as Record<string, unknown>;+ if (+ Array.isArray(schema.anyOf) ||+ Array.isArray(schema.oneOf) ||+ Array.isArray(schema.allOf)+ ) {+ return true;+ }++ return Object.values(schema).some(hasJsonSchemaComposition);+};++describe("MCP evaluator input", () => {+ it("exposes a plain JSON schema without unions or intersections", () => {+ const schema = z.toJSONSchema(McpEvaluatorInputBase, {+ target: "draft-7",+ unrepresentable: "any",+ });++ expect(schema.properties?.modelConfig).toMatchObject({+ type: "object",+ required: ["provider", "model"],+ });+ expect(hasJsonSchemaComposition(schema)).toBe(false);+ });++ it("derives a plain variable mapping schema from the shared schema", () => {+ expect(+ McpEvaluatorInput.safeParse({+ ...llmEvaluatorInput,+ variableMapping: [+ {+ templateVariable: "output",+ selectedColumnId: "output",+ jsonSelector: "$.answer",+ },+ ],+ }).success,+ ).toBe(true);+ });++ it("validates custom model configuration through its schema", () => {+ const incomplete = McpEvaluatorInput.safeParse({+ ...llmEvaluatorInput,+ modelConfig: { modelParams: { temperature: 0.2 } },+ });++ expect(incomplete.success).toBe(false);+ if (!incomplete.success) {+ expect(incomplete.error.issues.map(({ path }) => path)).toEqual(+ expect.arrayContaining([+ ["modelConfig", "provider"],+ ["modelConfig", "model"],+ ]),+ );+ }++ expect(+ McpEvaluatorInput.safeParse({+ ...llmEvaluatorInput,+ modelConfig: {+ provider: "openai",+ model: "gpt-4.1-mini",+ modelParams: {+ temperature: 0.2,+ providerOptions: { openai: { reasoningEffort: "low" } },+ },+ },+ }).success,+ ).toBe(true);+ });+});diff --git a/web/src/__tests__/server/unit/evaluator-v2-validation.servertest.ts b/web/src/__tests__/server/unit/evaluator-v2-validation.servertest.tsindex c935df44f2..d9fc6b4fae 100644--- a/web/src/__tests__/server/unit/evaluator-v2-validation.servertest.ts+++ b/web/src/__tests__/server/unit/evaluator-v2-validation.servertest.ts@@ -41,13 +41,9 @@ describe("evaluator configuration validation", () => {definition: {type: EvalTemplateType.LLM_AS_JUDGE,prompt: "Judge {{output}}",- provider: null,- model: null,- modelParams: null,- vars: ["output"],+ modelConfig: null,variableMapping: null,outputDefinition: {- version: 2,dataType: "NUMERIC",score: { description: "Quality" },reasoning: { description: "Reasoning" },@@ -111,7 +107,6 @@ describe("evaluator configuration validation", () => {vars: ["output"],variableMapping: null,outputDefinition: {- version: 2,dataType: "NUMERIC",score: { description: "Quality" },reasoning: { description: "Reasoning" },@@ -161,7 +156,6 @@ describe("evaluator configuration validation", () => {vars: ["output"],variableMapping: null,outputDefinition: {- version: 2,dataType: "NUMERIC",score: { description: "Quality" },reasoning: { description: "Reasoning" },@@ -191,7 +185,6 @@ describe("evaluator configuration validation", () => {},],outputDefinition: {- version: 2,dataType: "NUMERIC",score: { description: "Quality" },reasoning: { description: "Reasoning" },@@ -219,7 +212,6 @@ describe("evaluator configuration validation", () => {{ templateVariable: "output", selectedColumnId: "output" },],outputDefinition: {- version: 2,dataType: "NUMERIC",score: { description: "Quality" },reasoning: { description: "Reasoning" },@@ -249,7 +241,6 @@ describe("evaluator configuration validation", () => {},],outputDefinition: {- version: 2,dataType: "NUMERIC",score: { description: "Quality" },reasoning: { description: "Reasoning" },diff --git a/web/src/__tests__/server/unit/python-sdk-deprecations.servertest.ts b/web/src/__tests__/server/unit/python-sdk-deprecations.servertest.tsindex 6d92c99f44..6e5538249e 100644--- a/web/src/__tests__/server/unit/python-sdk-deprecations.servertest.ts+++ b/web/src/__tests__/server/unit/python-sdk-deprecations.servertest.ts@@ -118,6 +118,15 @@ describe("Python SDK deprecations", () => {"sessions:get","trace:get","trace:list",+ "unstable/evaluation_rules:create",+ "unstable/evaluation_rules:list",+ "unstable/evaluation_rules:get",+ "unstable/evaluation_rules:update",+ "unstable/evaluation_rules:delete",+ "unstable/evaluators:create",+ "unstable/evaluators:list",+ "unstable/evaluators:get",+ "unstable/evaluators:delete",]);});diff --git a/web/src/__tests__/server/unit/rate-limit-response.servertest.ts b/web/src/__tests__/server/unit/rate-limit-response.servertest.tsindex 785c12b850..5793c50a48 100644--- a/web/src/__tests__/server/unit/rate-limit-response.servertest.ts+++ b/web/src/__tests__/server/unit/rate-limit-response.servertest.ts@@ -3,10 +3,10 @@ import { createMocks } from "node-mocks-http";import type { RateLimitResult } from "@langfuse/shared";const {- mockCreateUnstablePublicApiRateLimitError,- mockSendUnstablePublicApiErrorResponse,+ mockCreateStructuredPublicApiRateLimitError,+ mockSendStructuredPublicApiErrorResponse,} = vi.hoisted(() => ({- mockCreateUnstablePublicApiRateLimitError: vi.fn(+ mockCreateStructuredPublicApiRateLimitError: vi.fn((rateLimitRes: RateLimitResult,options?: {@@ -33,7 +33,7 @@ const {};},),- mockSendUnstablePublicApiErrorResponse: vi.fn((res, error) =>+ mockSendStructuredPublicApiErrorResponse: vi.fn((res, error) =>res.status(error.httpCode).json({message: error.message,code: error.code,@@ -74,12 +74,13 @@ vi.mock("@langfuse/shared/src/server", () => ({}));vi.mock(- "@/src/features/public-api/server/unstable-public-api-error-contract",+ "@/src/features/public-api/server/structuredPublicApiErrorContract",() => ({- unstablePublicEvalsErrorContract: "unstable-public-evals",- createUnstablePublicApiRateLimitError:- mockCreateUnstablePublicApiRateLimitError,- sendUnstablePublicApiErrorResponse: mockSendUnstablePublicApiErrorResponse,+ structuredPublicApiErrorContract: "structured",+ createStructuredPublicApiRateLimitError:+ mockCreateStructuredPublicApiRateLimitError,+ sendStructuredPublicApiErrorResponse:+ mockSendStructuredPublicApiErrorResponse,}),);@@ -141,11 +142,11 @@ describe("sendRateLimitResponse", () => {expect(res.getHeader("Retry-After")).toBe(3);expect(res.getHeader("X-RateLimit-Limit")).toBe(10);expect(res.getHeader("X-RateLimit-Remaining")).toBe(-1);- expect(mockCreateUnstablePublicApiRateLimitError).toHaveBeenCalledWith(+ expect(mockCreateStructuredPublicApiRateLimitError).toHaveBeenCalledWith(rateLimitResult,{},);- expect(mockSendUnstablePublicApiErrorResponse).toHaveBeenCalledTimes(1);+ expect(mockSendStructuredPublicApiErrorResponse).toHaveBeenCalledTimes(1);});it("returns upgrade guidance for stable responses with an upgrade path", () => {@@ -168,31 +169,31 @@ describe("sendRateLimitResponse", () => {resetAt: expect.any(String),},});- expect(mockCreateUnstablePublicApiRateLimitError).toHaveBeenCalledWith(+ expect(mockCreateStructuredPublicApiRateLimitError).toHaveBeenCalledWith(rateLimitResult,{upgradePath,},);- expect(mockSendUnstablePublicApiErrorResponse).toHaveBeenCalledTimes(1);+ expect(mockSendStructuredPublicApiErrorResponse).toHaveBeenCalledTimes(1);});it("passes the error contract through when an upgrade path is present", () => {const res = createResponse();sendRateLimitResponse(res, rateLimitResult, {- errorContract: "unstable-public-evals",+ errorContract: "structured",upgradePath,});- expect(mockCreateUnstablePublicApiRateLimitError).toHaveBeenCalledWith(+ expect(mockCreateStructuredPublicApiRateLimitError).toHaveBeenCalledWith(rateLimitResult,{- errorContract: "unstable-public-evals",+ errorContract: "structured",upgradePath,},);- expect(mockSendUnstablePublicApiErrorResponse).toHaveBeenCalledTimes(1);+ expect(mockSendStructuredPublicApiErrorResponse).toHaveBeenCalledTimes(1);expect(res.statusCode).toBe(429);expect(res._getJSONData()).toEqual({message:diff --git a/web/src/__tests__/server/unit/unstable-public-api-error-contract.servertest.ts b/web/src/__tests__/server/unit/structured-public-api-error-contract.servertest.tssimilarity index 54%rename from web/src/__tests__/server/unit/unstable-public-api-error-contract.servertest.tsrename to web/src/__tests__/server/unit/structured-public-api-error-contract.servertest.tsindex 0f9c3a8ef7..d4223bf12b 100644--- a/web/src/__tests__/server/unit/unstable-public-api-error-contract.servertest.ts+++ b/web/src/__tests__/server/unit/structured-public-api-error-contract.servertest.ts@@ -1,7 +1,38 @@import type { RateLimitResult } from "@langfuse/shared";-import { createUnstablePublicApiRateLimitError } from "@/src/features/public-api/server/unstable-public-api-error-contract";+import { ClickHouseResourceError } from "@langfuse/shared/src/server";+import { EvaluatorVersionConflictError } from "@/src/features/evals/v2/server/evaluators/evaluatorErrors";+import {+ createStructuredPublicApiRateLimitError,+ structuredPublicApiErrorContract,+ toStructuredPublicApiError,+} from "@/src/features/public-api/server";++describe("structured public api error contract", () => {+ it("uses one contract marker", () => {+ expect(structuredPublicApiErrorContract).toBe("structured");+ });++ it("maps evaluator version conflicts to the conflict code", () => {+ const error = toStructuredPublicApiError(+ new EvaluatorVersionConflictError(),+ );++ expect(error.name).toBe("StructuredPublicApiError");+ expect(error.httpCode).toBe(409);+ expect(error.code).toBe("conflict");+ });++ it("only emits shared codes for mapped errors", () => {+ const resourceError = new ClickHouseResourceError(+ "TIMEOUT",+ new Error("timed out"),+ );++ expect(toStructuredPublicApiError(resourceError).code).toBe(+ "invalid_request",+ );+ });-describe("unstable public api error contract", () => {it("supports upgrade path rate limit messages and clamps remaining points", () => {const upgradePath = {legacyEndpoint: "GET /api/public/traces",@@ -30,7 +61,7 @@ describe("unstable public api error contract", () => {isFirstInDuration: false,} satisfies RateLimitResult;- const error = createUnstablePublicApiRateLimitError(rateLimitResult, {+ const error = createStructuredPublicApiRateLimitError(rateLimitResult, {upgradePath,});diff --git a/web/src/__tests__/server/unit/typescript-sdk-deprecations.servertest.ts b/web/src/__tests__/server/unit/typescript-sdk-deprecations.servertest.tsindex 9ee7ac7ca1..43c9afd798 100644--- a/web/src/__tests__/server/unit/typescript-sdk-deprecations.servertest.ts+++ b/web/src/__tests__/server/unit/typescript-sdk-deprecations.servertest.ts@@ -146,6 +146,15 @@ describe("TypeScript SDK deprecations", () => {"api/resources/sessions/client/Client.ts:get","api/resources/trace/client/Client.ts:get","api/resources/trace/client/Client.ts:list",+ "api/resources/unstable/resources/evaluationRules/client/Client.ts:create",+ "api/resources/unstable/resources/evaluationRules/client/Client.ts:list",+ "api/resources/unstable/resources/evaluationRules/client/Client.ts:get",+ "api/resources/unstable/resources/evaluationRules/client/Client.ts:update",+ "api/resources/unstable/resources/evaluationRules/client/Client.ts:delete",+ "api/resources/unstable/resources/evaluators/client/Client.ts:create",+ "api/resources/unstable/resources/evaluators/client/Client.ts:list",+ "api/resources/unstable/resources/evaluators/client/Client.ts:get",+ "api/resources/unstable/resources/evaluators/client/Client.ts:delete",]);expect(deprecationsdiff --git a/web/src/__tests__/server/unit/unstable-evals-contract.servertest.ts b/web/src/__tests__/server/unit/unstable-evals-contract.servertest.tsindex 408d8171da..bff8db6736 100644--- a/web/src/__tests__/server/unit/unstable-evals-contract.servertest.ts+++ b/web/src/__tests__/server/unit/unstable-evals-contract.servertest.ts@@ -11,7 +11,7 @@ import {toApiV2EvaluationRule,toEvaluationRuleInput,} from "@/src/features/evals/server/unstable-public-api/adapters";-import { UnstablePublicApiError } from "@/src/features/public-api/server/unstable-public-api-error-contract";+import { StructuredPublicApiError } from "@/src/features/public-api";import type {StoredPublicEvaluatorTemplate,StoredPublicV2EvaluationRule,@@ -37,21 +37,21 @@ const numericOutputDefinition = createNumericEvalOutputDefinition({maxValue: 1,});-const expectUnstablePublicApiError = (+const expectStructuredPublicApiError = (fn: () => unknown,params: {- code: UnstablePublicApiError["code"];+ code: StructuredPublicApiError["code"];message?: string;details?: Record<string, unknown>;},) => {try {fn();- throw new Error("Expected function to throw UnstablePublicApiError");+ throw new Error("Expected function to throw StructuredPublicApiError");} catch (error) {- expect(error).toBeInstanceOf(UnstablePublicApiError);+ expect(error).toBeInstanceOf(StructuredPublicApiError);- const unstableError = error as UnstablePublicApiError;+ const unstableError = error as StructuredPublicApiError;expect(unstableError.code).toBe(params.code);if (params.message) {@@ -659,7 +659,7 @@ describe("unstable public eval adapters", () => {});it("rejects invalid static filter option values", () => {- expectUnstablePublicApiError(+ expectStructuredPublicApiError(() =>toEvaluationRuleInput({input: {@@ -692,7 +692,7 @@ describe("unstable public eval adapters", () => {});it("rejects malformed jsonPath selectors", () => {- expectUnstablePublicApiError(+ expectStructuredPublicApiError(() =>toEvaluationRuleInput({input: {@@ -724,7 +724,7 @@ describe("unstable public eval adapters", () => {});it("rejects mapping sources that are incompatible with the selected target", () => {- expectUnstablePublicApiError(+ expectStructuredPublicApiError(() =>toEvaluationRuleInput({input: {@@ -778,7 +778,7 @@ describe("unstable public eval adapters", () => {});it("rejects missing evaluator variable mappings", () => {- expectUnstablePublicApiError(+ expectStructuredPublicApiError(() =>toEvaluationRuleInput({input: {diff --git a/web/src/__tests__/server/unstable-dashboard-widgets-api.servertest.ts b/web/src/__tests__/server/unstable-dashboard-widgets-api.servertest.tsindex 58e748898a..fa3f361771 100644--- a/web/src/__tests__/server/unstable-dashboard-widgets-api.servertest.ts+++ b/web/src/__tests__/server/unstable-dashboard-widgets-api.servertest.ts@@ -8,7 +8,7 @@ import {PatchUnstableDashboardWidgetResponse,PostUnstableDashboardWidgetResponse,} from "@/src/features/public-api/types/unstable-dashboard-widgets";-import { UnstablePublicApiErrorResponse } from "@/src/features/public-api/types/unstable-public-evals-contract";+import { StructuredPublicApiErrorResponse } from "@/src/features/public-api/types/unstable-public-evals-contract";import { DashboardWidgetViews, prisma } from "@langfuse/shared/src/db";import {createOrgProjectAndApiKey,@@ -41,11 +41,11 @@ const expectUnstableError = (response: Awaited<ReturnType<typeof makeAPICall>>,params: {status: number;- code: z.infer<typeof UnstablePublicApiErrorResponse>["code"];+ code: z.infer<typeof StructuredPublicApiErrorResponse>["code"];},) => {expect(response.status).toBe(params.status);- const body = UnstablePublicApiErrorResponse.parse(response.body);+ const body = StructuredPublicApiErrorResponse.parse(response.body);expect(body.code).toBe(params.code);return body;};diff --git a/web/src/__tests__/server/unstable-dashboards-api.servertest.ts b/web/src/__tests__/server/unstable-dashboards-api.servertest.tsindex 1dedeb6ad9..e9459c8694 100644--- a/web/src/__tests__/server/unstable-dashboards-api.servertest.ts+++ b/web/src/__tests__/server/unstable-dashboards-api.servertest.ts@@ -19,7 +19,7 @@ import {PostDashboardPlacementResponse,PostUnstableDashboardResponse,} from "@/src/features/public-api/types/unstable-dashboards";-import { UnstablePublicApiErrorResponse } from "@/src/features/public-api/types/unstable-public-evals-contract";+import { StructuredPublicApiErrorResponse } from "@/src/features/public-api/types/unstable-public-evals-contract";import { createOrgProjectAndApiKey } from "@langfuse/shared/src/server";import { prisma } from "@langfuse/shared/src/db";import { nanoid } from "nanoid";@@ -178,7 +178,7 @@ describe("unstable dashboard API", () => {);expect(duplicatePlacement.status).toBe(409);expect(- UnstablePublicApiErrorResponse.parse(duplicatePlacement.body).code,+ StructuredPublicApiErrorResponse.parse(duplicatePlacement.body).code,).toBe("conflict");const blockedDelete = await makeAPICall(@@ -188,9 +188,9 @@ describe("unstable dashboard API", () => {auth,);expect(blockedDelete.status).toBe(409);- expect(UnstablePublicApiErrorResponse.parse(blockedDelete.body).code).toBe(- "conflict",- );+ expect(+ StructuredPublicApiErrorResponse.parse(blockedDelete.body).code,+ ).toBe("conflict");const moved = await makeZodVerifiedAPICall(PatchDashboardPlacementResponse,diff --git a/web/src/__tests__/server/unstable-evals-api.servertest.ts b/web/src/__tests__/server/unstable-evals-api.servertest.tsindex 838afcebde..1b1c2d720a 100644--- a/web/src/__tests__/server/unstable-evals-api.servertest.ts+++ b/web/src/__tests__/server/unstable-evals-api.servertest.ts@@ -26,7 +26,7 @@ import {createOrgProjectAndApiKey,} from "@langfuse/shared/src/server";import { ApiKeyScope } from "@prisma/client";-import { UnstablePublicApiErrorResponse } from "@/src/features/public-api/types/unstable-public-evals-contract";+import { StructuredPublicApiErrorResponse } from "@/src/features/public-api/types/unstable-public-evals-contract";import type { z } from "zod";import { encrypt } from "@langfuse/shared/encryption";@@ -71,11 +71,11 @@ const expectUnstableError = (response: Awaited<ReturnType<typeof makeAPICall>>,params: {status: number;- code: z.infer<typeof UnstablePublicApiErrorResponse>["code"];+ code: z.infer<typeof StructuredPublicApiErrorResponse>["code"];},) => {expect(response.status).toBe(params.status);- const body = UnstablePublicApiErrorResponse.parse(response.body);+ const body = StructuredPublicApiErrorResponse.parse(response.body);expect(body.code).toBe(params.code);return body;};diff --git a/web/src/features/audit-logs/server/index.ts b/web/src/features/audit-logs/server/index.tsnew file mode 100644index 0000000000..4bd54c47e6--- /dev/null+++ b/web/src/features/audit-logs/server/index.ts@@ -0,0 +1 @@+export { auditLog } from "@/src/features/audit-logs/auditLog";diff --git a/web/src/features/evals/server/index.ts b/web/src/features/evals/server/index.tsnew file mode 100644index 0000000000..e8fdb7a0a3--- /dev/null+++ b/web/src/features/evals/server/index.ts@@ -0,0 +1,8 @@+export {+ EVAL_TEMPLATE_AUDIT_LOG_RESOURCE_TYPE,+ JOB_CONFIGURATION_AUDIT_LOG_RESOURCE_TYPE,+} from "./audit-log-resource-types";+export { EvaluatorService } from "@/src/features/evals/v2/server/evaluators/evaluatorService";+export { EvaluatorDefinitionInputSchema } from "@/src/features/evals/v2/server/evaluators/evaluatorTypes";+export { RuleService } from "@/src/features/evals/v2/server/rules/ruleService";+export { isLegacyEvalTarget } from "@/src/features/evals/utils/typeHelpers";diff --git a/web/src/features/evals/server/unstable-public-api/adapters.ts b/web/src/features/evals/server/unstable-public-api/adapters.tsindex 82a4f453d9..1cbb70820a 100644--- a/web/src/features/evals/server/unstable-public-api/adapters.ts+++ b/web/src/features/evals/server/unstable-public-api/adapters.ts@@ -24,17 +24,17 @@ import { EvalTemplateType } from "@langfuse/shared/src/db";import { logger } from "@langfuse/shared/src/server";import { z } from "zod";import {- LegacyEvaluationRuleMapping,+ LegacyPromptVariableMapping,PUBLIC_EVALUATOR_TYPE_CODE,PUBLIC_EVALUATOR_TYPE_LLM_AS_JUDGE,PublicEvaluationRuleFilter,type PublicEvaluationRuleFilterType,- type PublicEvaluationRuleMappingType,- type PublicEvaluationRuleReadMappingType,+ type PromptVariableMappingInputType,+ type PromptVariableMappingReadType,type PublicEvaluationRuleReadTargetType,type PublicEvaluationRuleStatusType,type PublicEvaluationRuleTargetType,- type LegacyEvaluationRuleMappingType,+ type LegacyPromptVariableMappingType,type PublicEvaluatorModelConfigType,type PublicEvaluatorOutputDefinitionType,type PublicEvaluatorTypeType,@@ -81,7 +81,7 @@ const INTERNAL_TARGET_OBJECT_TO_PUBLIC_TARGET: Record<};const PUBLIC_MAPPING_SOURCE_TO_INTERNAL_COLUMN: Record<- PublicEvaluationRuleMappingType["source"],+ PromptVariableMappingInputType["source"],ObservationVariableMapping["selectedColumnId"]> = {input: "input",@@ -94,7 +94,7 @@ const PUBLIC_MAPPING_SOURCE_TO_INTERNAL_COLUMN: Record<const INTERNAL_MAPPING_COLUMN_TO_PUBLIC_SOURCE: Record<string,- PublicEvaluationRuleMappingType["source"]+ PromptVariableMappingInputType["source"]> = {input: "input",output: "output",@@ -121,7 +121,7 @@ export function deriveEvaluatorVariables(}export function toStoredVariableMappings(params: {- mappings: PublicEvaluationRuleMappingType[];+ mappings: PromptVariableMappingInputType[];variables: string[];target: PublicEvaluationRuleTargetType;}) {@@ -140,7 +140,7 @@ export function toStoredVariableMappings(params: {* elsewhere; prefer `toStoredVariableMappings` when they are.*/export function toStoredMappingList(- mappings: PublicEvaluationRuleMappingType[],+ mappings: PromptVariableMappingInputType[],) {return observationVariableMappingList.parse(mappings.map((mapping) => ({@@ -297,7 +297,7 @@ function toApiFilter(filter: z.infer<typeof singleFilter>) {export function toApiReadMappings(mappings: unknown,-): PublicEvaluationRuleReadMappingType[] {+): PromptVariableMappingReadType[] {const parsed = observationVariableMappingList.safeParse(mappings);if (!parsed.success) {@@ -325,7 +325,7 @@ export function toApiReadMappings(export function toApiMappings(mappings: unknown,-): PublicEvaluationRuleMappingType[] {+): PromptVariableMappingInputType[] {return toApiReadMappings(mappings).map((mapping) => {if (mapping.source === null) {throw new InternalServerError("Evaluation rule mapping is corrupted");@@ -337,7 +337,7 @@ export function toApiMappings(function toApiLegacyMappings(mappings: unknown,-): LegacyEvaluationRuleMappingType[] {+): LegacyPromptVariableMappingType[] {const parsed = variableMappingList.safeParse(mappings);if (!parsed.success) {@@ -355,7 +355,7 @@ function toApiLegacyMappings(...(mapping.jsonSelector ? { jsonPath: mapping.jsonSelector } : {}),}));const parsedPublicMappings = z- .array(LegacyEvaluationRuleMapping)+ .array(LegacyPromptVariableMapping).safeParse(publicMappings);if (!parsedPublicMappings.success) {logger.error("Failed to parse unstable public legacy rule mappings", {@@ -369,7 +369,7 @@ function toApiLegacyMappings(function toDefaultLegacyMappings(mappings: unknown,target: "trace" | "dataset",-): LegacyEvaluationRuleMappingType[] {+): LegacyPromptVariableMappingType[] {return toApiMappings(mappings).map((mapping) => ({...mapping,langfuseObject: target === "trace" ? "trace" : "dataset_item",@@ -624,7 +624,7 @@ export function toEvaluationRuleInput(params: {enabled: boolean;sampling: number;filter: PublicEvaluationRuleFilterType[];- mapping?: PublicEvaluationRuleMappingType[];+ mapping?: PromptVariableMappingInputType[];};evaluatorVariables: string[];evaluatorType: PublicEvaluatorTypeType;@@ -667,7 +667,7 @@ export function toEvaluationRuleFields(input: {/** Stored mapping for one assignment. Code evaluators inherit their fixed evaluator mapping. */export function toStoredAssignmentMapping(params: {target: PublicEvaluationRuleTargetType;- mapping?: PublicEvaluationRuleMappingType[];+ mapping?: PromptVariableMappingInputType[];evaluatorVariables: string[];evaluatorType: PublicEvaluatorTypeType;}) {diff --git a/web/src/features/evals/server/unstable-public-api/evaluation-rule-service.ts b/web/src/features/evals/server/unstable-public-api/evaluation-rule-service.tsindex 40cef77bc6..c5161c67ba 100644--- a/web/src/features/evals/server/unstable-public-api/evaluation-rule-service.ts+++ b/web/src/features/evals/server/unstable-public-api/evaluation-rule-service.ts@@ -23,8 +23,8 @@ import type {PostUnstableEvaluationRuleBodyType,} from "@/src/features/public-api/types/unstable-evaluation-rules";import {- type PublicEvaluationRuleMappingType,- type PublicEvaluationRuleReadMappingType,+ type PromptVariableMappingInputType,+ type PromptVariableMappingReadType,type PublicEvaluationRuleTargetType,PUBLIC_EVALUATOR_TYPE_LLM_AS_JUDGE,} from "@/src/features/public-api/types/unstable-public-evals-contract";@@ -58,7 +58,7 @@ import {assertEvaluationRuleFilterValuesExistForProject,assertEvaluatorDefinitionCanRunForPublicApi,} from "./validation";-import { createUnstablePublicApiError } from "@/src/features/public-api/server/unstable-public-api-error-contract";+import { createStructuredPublicApiError } from "@/src/features/public-api";import { assertUnreachable } from "@/src/utils/types";const ruleService = new RuleService(prisma, async () => undefined);@@ -79,7 +79,7 @@ function toEvaluatorDefinition(): StoredPublicEvaluatorTemplate {const version = evaluator.versions[0];if (!version) {- throw createUnstablePublicApiError({+ throw createStructuredPublicApiError({httpCode: 500,code: "internal_error",message: "Evaluator version is missing",@@ -147,7 +147,7 @@ async function assertEvaluationRuleCanRunForPublicApi(params: {}if (!isCodeEvalEnabled()) {- throw createUnstablePublicApiError({+ throw createStructuredPublicApiError({httpCode: 403,code: "access_denied",message: "Code evals are not enabled",@@ -157,7 +157,7 @@ async function assertEvaluationRuleCanRunForPublicApi(params: {if (!isCodeEvalSourceCodeLanguageSupported(params.template.sourceCodeLanguage)) {- throw createUnstablePublicApiError({+ throw createStructuredPublicApiError({httpCode: 400,code: "invalid_request",message:@@ -183,21 +183,21 @@ async function assertEvaluationRuleCanRunForPublicApi(params: {switch (error.code) {case "invalid_target":case "invalid_request":- throw createUnstablePublicApiError({+ throw createStructuredPublicApiError({httpCode: 400,code: "invalid_request",message: error.message,details,});case "resource_not_found":- throw createUnstablePublicApiError({+ throw createStructuredPublicApiError({httpCode: 404,code: "resource_not_found",message: error.message,details,});case "preflight_failed":- throw createUnstablePublicApiError({+ throw createStructuredPublicApiError({httpCode: 422,code: "evaluator_preflight_failed",message: error.message,@@ -227,7 +227,7 @@ async function assertActivePublicApiEvaluationRuleLimitNotExceeded(function throwPublicRuleServiceError(error: unknown): never {if (error instanceof ActiveEvaluationRuleLimitError) {- throw createUnstablePublicApiError({+ throw createStructuredPublicApiError({httpCode: 409,code: "conflict",message: error.message,@@ -321,7 +321,7 @@ export async function createPublicEvaluationRule(params: {select: { id: true },});if (existing) {- throw createUnstablePublicApiError({+ throw createStructuredPublicApiError({httpCode: 409,code: "name_conflict",message: `An evaluation rule named "${params.input.name}" already exists in this project. Use PATCH /api/public/unstable/evaluation-rules/${existing.id} to update it instead of creating a duplicate.`,@@ -351,7 +351,7 @@ export async function createPublicEvaluationRule(params: {),);if (new Set(evaluators.map(({ id }) => id)).size !== evaluators.length) {- throw createUnstablePublicApiError({+ throw createStructuredPublicApiError({httpCode: 409,code: "conflict",message: "An evaluator can only be attached once to an evaluation rule.",@@ -424,7 +424,7 @@ async function createRuleWithRuleService(params: {data: ReturnType<typeof toEvaluationRuleInput>;assignments: Array<{evaluator: PublicV2Evaluator;- assignment: { mapping?: PublicEvaluationRuleMappingType[] };+ assignment: { mapping?: PromptVariableMappingInputType[] };data: ReturnType<typeof toEvaluationRuleInput>;}>;}) {@@ -466,9 +466,9 @@ type PreparedAssignment = {};function toWritableMappingsIfComplete(- mappings: PublicEvaluationRuleReadMappingType[],-): PublicEvaluationRuleMappingType[] | undefined {- const completeMappings: PublicEvaluationRuleMappingType[] = [];+ mappings: PromptVariableMappingReadType[],+): PromptVariableMappingInputType[] | undefined {+ const completeMappings: PromptVariableMappingInputType[] = [];for (const mapping of mappings) {if (mapping.source === null) return undefined;completeMappings.push({ ...mapping, source: mapping.source });@@ -484,7 +484,7 @@ async function prepareAssignment(params: {projectId: string;target: PublicEvaluationRuleTargetType;evaluator: EvaluationRuleEvaluatorFamilyReference;- mapping?: PublicEvaluationRuleMappingType[];+ mapping?: PromptVariableMappingInputType[];evaluatorId?: string;}): Promise<PreparedAssignment> {const evaluator = await resolveProjectEvaluator({@@ -523,7 +523,7 @@ async function prepareAssignment(params: {function assertUniqueEvaluators(assignments: Array<{ evaluatorId: string }>) {const ids = assignments.map(({ evaluatorId }) => evaluatorId);if (new Set(ids).size !== ids.length) {- throw createUnstablePublicApiError({+ throw createStructuredPublicApiError({httpCode: 409,code: "conflict",message: "An evaluator can only be attached once to an evaluation rule.",@@ -604,7 +604,7 @@ export async function updatePublicEvaluationRule(params: {params.input.mapping !== undefined &&firstAssignment?.evaluator.type === EvalTemplateType.CODE) {- throw createUnstablePublicApiError({+ throw createStructuredPublicApiError({httpCode: 400,code: "invalid_body",message:@@ -613,7 +613,7 @@ export async function updatePublicEvaluationRule(params: {});}if (patchesFirstAssignment && !firstAssignment && !params.input.evaluator) {- throw createUnstablePublicApiError({+ throw createStructuredPublicApiError({httpCode: 400,code: "invalid_body",message:diff --git a/web/src/features/evals/server/unstable-public-api/evaluator-service.ts b/web/src/features/evals/server/unstable-public-api/evaluator-service.tsindex c15794d313..b41db5975f 100644--- a/web/src/features/evals/server/unstable-public-api/evaluator-service.ts+++ b/web/src/features/evals/server/unstable-public-api/evaluator-service.ts@@ -21,7 +21,7 @@ import {EvaluatorVersionConflictError,} from "@/src/features/evals/v2/server/evaluators/evaluatorErrors";import { RuleService } from "@/src/features/evals/v2/server/rules/ruleService";-import { createUnstablePublicApiError } from "@/src/features/public-api/server/unstable-public-api-error-contract";+import { createStructuredPublicApiError } from "@/src/features/public-api";import type { PostUnstableEvaluatorBodyParsedType } from "@/src/features/public-api/types/unstable-evaluators";import { PUBLIC_EVALUATOR_TYPE_CODE } from "@/src/features/public-api/types/unstable-public-evals-contract";import {@@ -74,7 +74,7 @@ function assertCodeEvaluatorDefinitionCanRunForPublicApi(>,) {if (!isCodeEvalEnabled()) {- throw createUnstablePublicApiError({+ throw createStructuredPublicApiError({httpCode: 403,code: "access_denied",message: "Code evals are not enabled",@@ -82,7 +82,7 @@ function assertCodeEvaluatorDefinitionCanRunForPublicApi(}if (!isCodeEvalSourceCodeLanguageSupported(input.sourceCodeLanguage)) {- throw createUnstablePublicApiError({+ throw createStructuredPublicApiError({httpCode: 400,code: "invalid_request",message:@@ -125,14 +125,16 @@ function toDefinition(};}-type StableEvaluator = Awaited<ReturnType<EvaluatorService["get"]>>;+type StableEvaluator =+ | Awaited<ReturnType<EvaluatorService["get"]>>+ | Awaited<ReturnType<EvaluatorService["list"]>>["evaluators"][number];function toLatestTemplate(evaluator: StableEvaluator,): StoredPublicEvaluatorTemplate {const version = evaluator.versions[0];if (!version) {- throw createUnstablePublicApiError({+ throw createStructuredPublicApiError({httpCode: 500,code: "internal_error",message: "Evaluator version is missing",@@ -240,7 +242,7 @@ export async function createPublicEvaluator(params: {return toPublicEvaluator(evaluator, ruleCounts[evaluator.id] ?? 0);} catch (error) {if (error instanceof EvaluatorConfigurationError) {- throw createUnstablePublicApiError({+ throw createStructuredPublicApiError({httpCode: 422,code: "evaluator_preflight_failed",message: error.message,@@ -258,14 +260,14 @@ export async function createPublicEvaluator(params: {});}if (error instanceof EvaluatorVersionConflictError) {- throw createUnstablePublicApiError({+ throw createStructuredPublicApiError({httpCode: 409,code: "conflict",message: error.message,});}if (error instanceof LangfuseConflictError) {- throw createUnstablePublicApiError({+ throw createStructuredPublicApiError({httpCode: 409,code: "name_conflict",message: error.message,diff --git a/web/src/features/evals/server/unstable-public-api/types.ts b/web/src/features/evals/server/unstable-public-api/types.tsindex fcf9af67cf..0f6c779521 100644--- a/web/src/features/evals/server/unstable-public-api/types.ts+++ b/web/src/features/evals/server/unstable-public-api/types.ts@@ -10,10 +10,10 @@ import type {PublicEvaluationRuleEvaluatorReferenceType,PublicEvaluationRuleEvaluatorType,PublicEvaluationRuleFilterType,- PublicEvaluationRuleReadMappingType,+ PromptVariableMappingReadType,PublicEvaluationRuleStatusType,PublicEvaluationRuleTargetType,- LegacyEvaluationRuleMappingType,+ LegacyPromptVariableMappingType,PublicEvaluationRuleLegacyTargetType,PublicEvaluatorModelConfigType,PublicEvaluatorOutputDefinitionType,@@ -37,7 +37,7 @@ type ApiEvaluatorRecordBase = {version: number;variables: string[];// Read shape: an evaluator default can be incomplete or name experiment-only sources.- mapping: PublicEvaluationRuleReadMappingType[] | null;+ mapping: PromptVariableMappingReadType[] | null;evaluationRuleCount: number;createdAt: Date;updatedAt: Date;@@ -85,23 +85,23 @@ type ApiEvaluationRuleRecordBase = {export type ApiWritableEvaluationRuleRecord = ApiEvaluationRuleRecordBase & {evaluators: Array<{evaluator: PublicEvaluationRuleEvaluatorType;- mapping: PublicEvaluationRuleReadMappingType[] | null;+ mapping: PromptVariableMappingReadType[] | null;}>;target: PublicEvaluationRuleTargetType;filter: PublicEvaluationRuleFilterType[];- mapping: PublicEvaluationRuleReadMappingType[];+ mapping: PromptVariableMappingReadType[];};type ApiLegacyEvaluationRuleRecord = ApiEvaluationRuleRecordBase & {evaluators: Array<{evaluator: PublicEvaluationRuleEvaluatorType;- mapping: LegacyEvaluationRuleMappingType[] | null;+ mapping: LegacyPromptVariableMappingType[] | null;}>;target: PublicEvaluationRuleLegacyTargetType;delay: number;timeScope: JobTimeScope[];filter: FilterCondition[];- mapping: LegacyEvaluationRuleMappingType[];+ mapping: LegacyPromptVariableMappingType[];};export type ApiEvaluationRuleRecord =diff --git a/web/src/features/evals/server/unstable-public-api/validation.ts b/web/src/features/evals/server/unstable-public-api/validation.tsindex 2a01e00810..8cb2056dae 100644--- a/web/src/features/evals/server/unstable-public-api/validation.ts+++ b/web/src/features/evals/server/unstable-public-api/validation.ts@@ -6,15 +6,15 @@ import { prisma } from "@langfuse/shared/src/db";import { JSONPath } from "jsonpath-plus";import {EXPERIMENT_EVALUATION_RULE_FILTER_COLUMNS,- ExperimentEvaluationRuleMappingSource,+ ExperimentPromptVariableMappingSource,OBSERVATION_EVALUATION_RULE_FILTER_COLUMNS,- ObservationEvaluationRuleMappingSource,+ ObservationPromptVariableMappingSource,type PublicEvaluationRuleFilterType,- type PublicEvaluationRuleMappingType,+ type PromptVariableMappingInputType,type PublicEvaluationRuleTargetType,} from "@/src/features/public-api/types/unstable-public-evals-contract";import { getEvaluatorDefinitionPreflightError } from "@/src/features/evals/server/evaluator-preflight";-import { createUnstablePublicApiError } from "@/src/features/public-api/server/unstable-public-api-error-contract";+import { createStructuredPublicApiError } from "@/src/features/public-api";// Derived from the same column lists as the request schemas, so this runtime// check cannot drift from the documented contract.@@ -66,8 +66,8 @@ const SUPPORTED_FILTER_COLUMNS_BY_TARGET = {// also guards paths that parse the target-agnostic mapping union) can never// drift from the documented contract.const SUPPORTED_MAPPING_SOURCES_BY_TARGET = {- observation: new Set<string>(ObservationEvaluationRuleMappingSource.options),- experiment: new Set<string>(ExperimentEvaluationRuleMappingSource.options),+ observation: new Set<string>(ObservationPromptVariableMappingSource.options),+ experiment: new Set<string>(ExperimentPromptVariableMappingSource.options),} as const satisfies Record<PublicEvaluationRuleTargetType, Set<string>>;export function validateEvaluationRuleFilters(params: {@@ -79,7 +79,7 @@ export function validateEvaluationRuleFilters(params: {for (const [filterIndex, filter] of params.filters.entries()) {if (!supportedColumns.has(filter.column)) {- throw createUnstablePublicApiError({+ throw createStructuredPublicApiError({httpCode: 400,code: "invalid_filter_value",message: `Filter column "${filter.column}" is not supported for target "${params.target}"`,@@ -106,7 +106,7 @@ export function validateEvaluationRuleFilters(params: {);if (invalidValues.length > 0) {- throw createUnstablePublicApiError({+ throw createStructuredPublicApiError({httpCode: 400,code: "invalid_filter_value",message: `Filter column "${filter.column}" contains unsupported value(s): ${invalidValues.join(", ")}`,@@ -170,7 +170,7 @@ export async function assertEvaluationRuleFilterValuesExistForProject(params: {);if (invalidValues.length > 0) {- throw createUnstablePublicApiError({+ throw createStructuredPublicApiError({httpCode: 400,code: "invalid_filter_value",message: `Filter column "datasetId" contains dataset id(s) that do not exist in this project: ${invalidValues.join(", ")}`,@@ -192,7 +192,7 @@ function validateJsonPath(params: {const { jsonPath, variable, mappingIndex } = params;if (!jsonPath.startsWith("$")) {- throw createUnstablePublicApiError({+ throw createStructuredPublicApiError({httpCode: 400,code: "invalid_json_path",message: `Mapping for variable "${variable}" has an invalid jsonPath "${jsonPath}". JSONPath expressions must start with "$".`,@@ -241,7 +241,7 @@ function validateJsonPath(params: {const expectedDelimiter = character === "]" ? "[" : "(";if (delimiterStack.pop() !== expectedDelimiter) {- throw createUnstablePublicApiError({+ throw createStructuredPublicApiError({httpCode: 400,code: "invalid_json_path",message: `Mapping for variable "${variable}" has an invalid jsonPath "${jsonPath}". JSONPath expressions must use balanced brackets and parentheses.`,@@ -256,7 +256,7 @@ function validateJsonPath(params: {}if (openQuote || delimiterStack.length > 0) {- throw createUnstablePublicApiError({+ throw createStructuredPublicApiError({httpCode: 400,code: "invalid_json_path",message: `Mapping for variable "${variable}" has an invalid jsonPath "${jsonPath}". JSONPath expressions must use balanced quotes, brackets, and parentheses.`,@@ -276,7 +276,7 @@ function validateJsonPath(params: {});} catch (error) {const message = error instanceof Error ? error.message : "Unknown error";- throw createUnstablePublicApiError({+ throw createStructuredPublicApiError({httpCode: 400,code: "invalid_json_path",message: `Mapping for variable "${variable}" has an invalid jsonPath "${jsonPath}". ${message}`,@@ -290,7 +290,7 @@ function validateJsonPath(params: {}export function validateEvaluatorVariableMappings(params: {- mappings: PublicEvaluationRuleMappingType[];+ mappings: PromptVariableMappingInputType[];variables: string[];target: PublicEvaluationRuleTargetType;}) {@@ -300,7 +300,7 @@ export function validateEvaluatorVariableMappings(params: {for (const [mappingIndex, mapping] of params.mappings.entries()) {if (!allowedSources.has(mapping.source)) {- throw createUnstablePublicApiError({+ throw createStructuredPublicApiError({httpCode: 400,code: "invalid_variable_mapping",message: `Mapping source "${mapping.source}" is not supported for target "${params.target}"`,@@ -313,7 +313,7 @@ export function validateEvaluatorVariableMappings(params: {}if (!variableSet.has(mapping.variable)) {- throw createUnstablePublicApiError({+ throw createStructuredPublicApiError({httpCode: 400,code: "invalid_variable_mapping",message: `Mapping variable "${mapping.variable}" is not present in the evaluator prompt`,@@ -325,7 +325,7 @@ export function validateEvaluatorVariableMappings(params: {}if (mappedVariables.has(mapping.variable)) {- throw createUnstablePublicApiError({+ throw createStructuredPublicApiError({httpCode: 400,code: "duplicate_variable_mapping",message: `Mapping variable "${mapping.variable}" can only be mapped once`,@@ -344,7 +344,7 @@ export function validateEvaluatorVariableMappings(params: {);if (missingVariables.length > 0) {- throw createUnstablePublicApiError({+ throw createStructuredPublicApiError({httpCode: 400,code: "missing_variable_mapping",message: `Missing mappings for evaluator variables: ${missingVariables.join(", ")}`,@@ -379,7 +379,7 @@ export async function assertEvaluatorDefinitionCanRunForPublicApi(params: {const error = await getEvaluatorDefinitionPreflightError(params);if (error) {- throw createUnstablePublicApiError({+ throw createStructuredPublicApiError({httpCode: 422,code: "evaluator_preflight_failed",message: error,diff --git a/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.stories.tsx b/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.stories.tsxindex 138d2090c7..cd358acc69 100644--- a/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.stories.tsx+++ b/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.stories.tsx@@ -59,7 +59,6 @@ const managedTemplate = {prompt: "Rate the relevance of {{generation}} to {{query}}.",variables: [{ name: "query", defaultMapping: { field: "input" } }],outputDefinition: {- version: 2,dataType: "NUMERIC",score: { description: "Relevance." },reasoning: { description: "One sentence." },@@ -104,7 +103,6 @@ const sections: GallerySection[] = [evaluator: {...managedTemplate.evaluator,outputDefinition: {- version: 2,dataType: "CATEGORICAL",score: {description: "Intent.",@@ -124,7 +122,6 @@ const sections: GallerySection[] = [evaluator: {...managedTemplate.evaluator,outputDefinition: {- version: 2,dataType: "BOOLEAN",score: { description: "Out of scope." },reasoning: { description: "One sentence." },diff --git a/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGallerySection/EvaluatorGallerySection.stories.tsx b/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGallerySection/EvaluatorGallerySection.stories.tsxindex 3b3a9d1d10..53edae18ee 100644--- a/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGallerySection/EvaluatorGallerySection.stories.tsx+++ b/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGallerySection/EvaluatorGallerySection.stories.tsx@@ -41,7 +41,6 @@ const template = {prompt: "Rate the relevance of {{generation}} to {{query}}.",variables: [{ name: "query", defaultMapping: { field: "input" } }],outputDefinition: {- version: 2,dataType: "NUMERIC",score: { description: "Relevance." },reasoning: { description: "One sentence." },diff --git a/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGallerySection/components/EvaluatorRecommendedCard/EvaluatorRecommendedCard.stories.tsx b/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGallerySection/components/EvaluatorRecommendedCard/EvaluatorRecommendedCard.stories.tsxindex 8685380bf4..1995978c73 100644--- a/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGallerySection/components/EvaluatorRecommendedCard/EvaluatorRecommendedCard.stories.tsx+++ b/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGallerySection/components/EvaluatorRecommendedCard/EvaluatorRecommendedCard.stories.tsx@@ -18,7 +18,6 @@ const template = {prompt: "Classify {{input}}.",variables: [{ name: "input", defaultMapping: { field: "input" } }],outputDefinition: {- version: 2,dataType: "CATEGORICAL",score: {description: "Intent.",diff --git a/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGallerySection/components/EvaluatorTemplateRow/EvaluatorTemplateRow.stories.tsx b/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGallerySection/components/EvaluatorTemplateRow/EvaluatorTemplateRow.stories.tsxindex 46ac4975c2..c983af03f2 100644--- a/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGallerySection/components/EvaluatorTemplateRow/EvaluatorTemplateRow.stories.tsx+++ b/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGallerySection/components/EvaluatorTemplateRow/EvaluatorTemplateRow.stories.tsx@@ -21,7 +21,6 @@ const managedTemplate = {prompt: "Rate the relevance of {{generation}} to {{query}}.",variables: [{ name: "query", defaultMapping: { field: "input" } }],outputDefinition: {- version: 2,dataType: "NUMERIC",score: { description: "Relevance." },reasoning: { description: "One sentence." },diff --git a/web/src/features/evals/v2/components/EvaluatorsEmptyState/components/EvaluatorsEmptyStateView/EvaluatorsEmptyStateView.stories.tsx b/web/src/features/evals/v2/components/EvaluatorsEmptyState/components/EvaluatorsEmptyStateView/EvaluatorsEmptyStateView.stories.tsxindex 5848847adf..c5db15b77b 100644--- a/web/src/features/evals/v2/components/EvaluatorsEmptyState/components/EvaluatorsEmptyStateView/EvaluatorsEmptyStateView.stories.tsx+++ b/web/src/features/evals/v2/components/EvaluatorsEmptyState/components/EvaluatorsEmptyStateView/EvaluatorsEmptyStateView.stories.tsx@@ -22,7 +22,6 @@ const startingPoints = [prompt: "Classify {{input}}.",variables: [{ name: "input", defaultMapping: { field: "input" } }],outputDefinition: {- version: 2,dataType: "CATEGORICAL",score: {description: "Topic.",@@ -54,7 +53,6 @@ const startingPoints = [{ name: "last_user_message", defaultMapping: { field: "input" } },],outputDefinition: {- version: 2,dataType: "BOOLEAN",score: { description: "True if disagreement is present." },reasoning: { description: "One sentence." },diff --git a/web/src/features/evals/v2/constants/managedTemplatesCatalog.ts b/web/src/features/evals/v2/constants/managedTemplatesCatalog.tsindex 2d5e42f2f0..25afb94cd8 100644--- a/web/src/features/evals/v2/constants/managedTemplatesCatalog.ts+++ b/web/src/features/evals/v2/constants/managedTemplatesCatalog.ts@@ -107,7 +107,6 @@ Replace these examples with your own taxonomy before use:User message: {{input}}`,variables: [{ name: "input", defaultMapping: { field: "input" } }],outputDefinition: {- version: 2,dataType: "CATEGORICAL",score: {description: "Predicted category.",@@ -193,7 +192,6 @@ Think step by step and return the structured result.`,{ name: "last_user_message", defaultMapping: { field: "input" } },],outputDefinition: {- version: 2,dataType: "BOOLEAN",score: {description: "Boolean verdict.",@@ -257,7 +255,6 @@ Last user message: {{last_user_message}}`,{ name: "last_user_message", defaultMapping: { field: "input" } },],outputDefinition: {- version: 2,dataType: "BOOLEAN",score: {description: "Boolean verdict.",@@ -412,7 +409,6 @@ Last user message: {{last_user_message}}`,{ name: "last_user_message", defaultMapping: { field: "input" } },],outputDefinition: {- version: 2,dataType: "BOOLEAN",score: {description: "True if distress is present, false otherwise.",@@ -477,7 +473,6 @@ Expected output: {{expected_output}}`,},],outputDefinition: {- version: 2,dataType: "BOOLEAN",score: {description: "Boolean verdict.",@@ -730,7 +725,6 @@ Assistant output: {{assistant_output}}`,{ name: "assistant_output", defaultMapping: { field: "output" } },],outputDefinition: {- version: 2,dataType: "CATEGORICAL",score: {description: "Relevance label.",@@ -782,7 +776,6 @@ Assistant output: {{assistant_output}}`,{ name: "assistant_output", defaultMapping: { field: "output" } },],outputDefinition: {- version: 2,dataType: "BOOLEAN",score: { description: "Boolean verdict." },reasoning: {@@ -834,7 +827,6 @@ Replace these example definitions with your own taxonomy before use:Input: {{input}}`,variables: [{ name: "input", defaultMapping: { field: "input" } }],outputDefinition: {- version: 2,dataType: "CATEGORICAL",score: {description: "Predicted category label.",@@ -881,7 +873,6 @@ You will receive an input and must assign exactly one primary language category.Input: {{input}}`,variables: [{ name: "input", defaultMapping: { field: "input" } }],outputDefinition: {- version: 2,dataType: "CATEGORICAL",score: {description: "Predicted category.",@@ -950,7 +941,6 @@ Context: {{context}}`,{ name: "context", defaultMapping: { field: "input" } },],outputDefinition: {- version: 2,dataType: "CATEGORICAL",score: {description: "Groundedness label.",@@ -1003,7 +993,6 @@ Context: {{context}}`,{ name: "context", defaultMapping: { field: "input" } },],outputDefinition: {- version: 2,dataType: "CATEGORICAL",score: {description: "Context precision label.",@@ -1062,7 +1051,6 @@ Context: {{context}}`,{ name: "context", defaultMapping: { field: "input" } },],outputDefinition: {- version: 2,dataType: "CATEGORICAL",score: {description: "Context recall label.",@@ -1127,7 +1115,6 @@ True if sensitive, identifying personal information appears in the output, falseAssistant output: {{output}}`,variables: [{ name: "output", defaultMapping: { field: "output" } }],outputDefinition: {- version: 2,dataType: "BOOLEAN",score: {description: "Boolean verdict.",@@ -1174,7 +1161,6 @@ Assistant output: {{assistant_output}}`,{ name: "assistant_output", defaultMapping: { field: "output" } },],outputDefinition: {- version: 2,dataType: "BOOLEAN",score: { description: "Boolean verdict." },reasoning: {@@ -1225,7 +1211,6 @@ True if prompt injection is detected, false otherwise.Input text: {{input_text}}`,variables: [{ name: "input_text", defaultMapping: { field: "input" } }],outputDefinition: {- version: 2,dataType: "BOOLEAN",score: {description: "Boolean verdict.",@@ -1279,7 +1264,6 @@ Classify the user's primary requested outcome into exactly one engineering taskTask text: {{task_text}}`,variables: [{ name: "task_text", defaultMapping: { field: "input" } }],outputDefinition: {- version: 2,dataType: "CATEGORICAL",score: {description: "Engineering task-type label.",@@ -1344,7 +1328,6 @@ Classify the task into the one department it most likely serves.Task text: {{task_text}}`,variables: [{ name: "task_text", defaultMapping: { field: "input" } }],outputDefinition: {- version: 2,dataType: "CATEGORICAL",score: {description: "Department usage label.",diff --git a/web/src/features/evals/v2/fns/evaluators/prepareEvaluatorDraft.clienttest.ts b/web/src/features/evals/v2/fns/evaluators/prepareEvaluatorDraft.clienttest.tsindex ec83f9f06e..0b374e90ac 100644--- a/web/src/features/evals/v2/fns/evaluators/prepareEvaluatorDraft.clienttest.ts+++ b/web/src/features/evals/v2/fns/evaluators/prepareEvaluatorDraft.clienttest.ts@@ -14,7 +14,6 @@ describe("prepareEvaluatorDraft", () => {vars: ["output"],variableMapping: [],outputDefinition: {- version: 2 as const,dataType: ScoreDataTypeEnum.NUMERIC,score: { description: "Quality" },reasoning: { description: "Reasoning" },@@ -48,10 +47,11 @@ describe("prepareEvaluatorDraft", () => {definition: {type: "LLM_AS_JUDGE",prompt: "Judge {{output}}",- provider: "openai",- model: "gpt-test",- modelParams: { temperature: 0.2 },- vars: ["output"],+ modelConfig: {+ provider: "openai",+ model: "gpt-test",+ modelParams: { temperature: 0.2 },+ },variableMapping: [{templateVariable: "output",diff --git a/web/src/features/evals/v2/fns/evaluators/prepareEvaluatorDraft.ts b/web/src/features/evals/v2/fns/evaluators/prepareEvaluatorDraft.tsindex 97d91b2f02..20a5b8454a 100644--- a/web/src/features/evals/v2/fns/evaluators/prepareEvaluatorDraft.ts+++ b/web/src/features/evals/v2/fns/evaluators/prepareEvaluatorDraft.ts@@ -25,24 +25,20 @@ export function prepareEvaluatorDraft(params: EvaluatorSetupDraftState) {variableFields: params.variableFields,}): [];- const variables = mappings.map(({ variable }) => variable);const definition =params.type === "LLM_AS_JUDGE"? outputDefinition? {type: params.type,prompt: params.prompt,- provider:- params.modelMode === "custom"- ? (params.selectedModel?.provider ?? null)+ modelConfig:+ params.modelMode === "custom" && params.selectedModel+ ? {+ provider: params.selectedModel.provider,+ model: params.selectedModel.model,+ modelParams: params.modelParams,+ }: null,- model:- params.modelMode === "custom"- ? (params.selectedModel?.model ?? null)- : null,- modelParams:- params.modelMode === "custom" ? params.modelParams : null,- vars: variables,variableMapping: mappings.map(({ variable, fieldState }) => ({templateVariable: variable,selectedColumnId: fieldState.selectedColumnId,diff --git a/web/src/features/evals/v2/fns/scoreOutput/buildScoreOutputDefinition.ts b/web/src/features/evals/v2/fns/scoreOutput/buildScoreOutputDefinition.tsindex a548eb1092..4f55872a0e 100644--- a/web/src/features/evals/v2/fns/scoreOutput/buildScoreOutputDefinition.ts+++ b/web/src/features/evals/v2/fns/scoreOutput/buildScoreOutputDefinition.ts@@ -1,7 +1,7 @@import {- PersistedEvalOutputDefinitionSchema,+ EvalOutputDefinitionSchema,ScoreDataTypeEnum,- type PersistedEvalOutputDefinition,+ type EvalOutputDefinition,} from "@langfuse/shared";import type { ScoreOutputFormState } from "@/src/features/evals/v2/scoreOutputTypes";import {@@ -18,9 +18,8 @@ function parseNumericBound(key: "minValue" | "maxValue", raw: string) {export function buildScoreOutputDefinition(state: ScoreOutputFormState,-): PersistedEvalOutputDefinition | null {+): EvalOutputDefinition | null {const base = {- version: 2 as const,dataType: state.dataType,reasoning: {description:@@ -52,6 +51,6 @@ export function buildScoreOutputDefinition(...base,score: { description: scoreDescription },};- const parsed = PersistedEvalOutputDefinitionSchema.safeParse(candidate);+ const parsed = EvalOutputDefinitionSchema.safeParse(candidate);return parsed.success ? parsed.data : null;}diff --git a/web/src/features/evals/v2/fns/templateGallery/visibleGallerySections.clienttest.ts b/web/src/features/evals/v2/fns/templateGallery/visibleGallerySections.clienttest.tsindex ae8093333c..444e7d6cc3 100644--- a/web/src/features/evals/v2/fns/templateGallery/visibleGallerySections.clienttest.ts+++ b/web/src/features/evals/v2/fns/templateGallery/visibleGallerySections.clienttest.ts@@ -22,7 +22,6 @@ const template = {prompt: "Rate relevance.",variables: [{ name: "query", defaultMapping: { field: "input" } }],outputDefinition: {- version: 2,dataType: "NUMERIC",score: { description: "Relevance." },reasoning: { description: "One sentence." },diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorErrors.ts b/web/src/features/evals/v2/server/evaluators/evaluatorErrors.tsindex 68d469e699..f7152cd46b 100644--- a/web/src/features/evals/v2/server/evaluators/evaluatorErrors.ts+++ b/web/src/features/evals/v2/server/evaluators/evaluatorErrors.ts@@ -1,13 +1,8 @@-import { BaseError } from "@langfuse/shared";+import { BaseError, LangfuseConflictError } from "@langfuse/shared";-export class EvaluatorVersionConflictError extends BaseError {+export class EvaluatorVersionConflictError extends LangfuseConflictError {constructor() {- super(- "EvaluatorVersionConflictError",- 409,- "Evaluator was updated concurrently. Retry the request.",- true,- );+ super("Evaluator was updated concurrently. Retry the request.");}}diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorRepository.ts b/web/src/features/evals/v2/server/evaluators/evaluatorRepository.tsindex 91782e64bd..04a172b887 100644--- a/web/src/features/evals/v2/server/evaluators/evaluatorRepository.ts+++ b/web/src/features/evals/v2/server/evaluators/evaluatorRepository.ts@@ -27,15 +27,36 @@ import { creatorOptionsWhere, creatorWhere } from "../creatorFilterPrisma";type PrismaTransaction = Prisma.TransactionClient;+const createdByUser = {+ select: { id: true, name: true, email: true },+} as const;+const latestVersion = {orderBy: { version: "desc" as const },take: 1,};+const latestVersionWithCreator = {+ ...latestVersion,+ include: { createdByUser },+};+const eventEvaluatorFilterColumnIds = new Set(eventsEvalFilterColumns.map((column) => column.id),);+const publicEvaluationRuleAssignments = (projectId: string) =>+ ({+ where: { projectId },+ orderBy: { createdAt: "desc" },+ select: {+ variableMapping: true,+ evaluationRule: {+ select: { id: true, targetObject: true },+ },+ },+ }) satisfies Prisma.EvaluationRuleEvaluatorAssignmentFindManyArgs;+export const batchEligibleEvaluatorWhere = {// Trace/dataset assignments carry rule-specific mappings that an// observation batch cannot resolve when it addresses the evaluator alone.@@ -264,8 +285,8 @@ export async function listEvaluators(params: {skip: (params.page - 1) * params.limit,take: params.limit,include: {- versions: latestVersion,- createdByUser: { select: { name: true, email: true } },+ versions: latestVersionWithCreator,+ createdByUser,_count: {select: {assignments: { where: { projectId: params.projectId } },@@ -274,7 +295,7 @@ export async function listEvaluators(params: {assignments: {where: { projectId: params.projectId },select: {- evaluationRule: { select: { id: true, status: true } },+ evaluationRule: { select: { id: true, name: true, status: true } },},},},@@ -288,6 +309,7 @@ export async function listEvaluators(params: {return {evaluators: evaluators.map(({ assignments, ...evaluator }) => ({...evaluator,+ assignments,assignedRuleIds: assignments.map(({ evaluationRule }) => evaluationRule.id,),@@ -303,6 +325,49 @@ export async function listEvaluators(params: {};}+export async function listEvaluatorsCursor(params: {+ prisma: PrismaClient;+ projectId: string;+ limit: number;+ cursor?: { createdAt: Date; id: string };+}) {+ const baseWhere = await evaluatorWhere(params);+ const where: Prisma.EvaluatorWhereInput = params.cursor+ ? {+ AND: [+ baseWhere,+ {+ OR: [+ { createdAt: { lt: params.cursor.createdAt } },+ {+ createdAt: params.cursor.createdAt,+ id: { lt: params.cursor.id },+ },+ ],+ },+ ],+ }+ : baseWhere;+ const records = await params.prisma.evaluator.findMany({+ where,+ orderBy: [{ createdAt: "desc" }, { id: "desc" }],+ take: params.limit + 1,+ include: {+ versions: latestVersionWithCreator,+ createdByUser,+ assignments: publicEvaluationRuleAssignments(params.projectId),+ },+ });+ const hasMore = records.length > params.limit;+ const evaluators = records.slice(0, params.limit);+ const last = evaluators.at(-1);+ return {+ evaluators,+ nextCursor:+ hasMore && last ? { createdAt: last.createdAt, id: last.id } : undefined,+ };+}+export async function listEvaluatorFilterOptions(params: {prisma: PrismaClient;projectId: string;@@ -313,7 +378,7 @@ export async function listEvaluatorFilterOptions(params: {select: {name: true,type: true,- createdByUser: { select: { name: true, email: true } },+ createdByUser,versions: {...latestVersion,select: { model: true },@@ -394,7 +459,7 @@ export async function listEvaluatorOptions(params: {name: true,type: true,updatedAt: true,- createdByUser: { select: { name: true, email: true } },+ createdByUser,blockedAt: true,versions: {orderBy: { version: "desc" },@@ -423,7 +488,9 @@ export function findEvaluator(params: {return params.prisma.evaluator.findFirst({where: { id: params.evaluatorId, projectId: params.projectId },include: {- versions: latestVersion,+ versions: latestVersionWithCreator,+ createdByUser,+ assignments: publicEvaluationRuleAssignments(params.projectId),},});}@@ -493,7 +560,8 @@ export async function listEvaluatorVersions(params: {orderBy: { version: "desc" },take: params.limit + 1,include: {- createdByUser: { select: { name: true, email: true } },+ createdByUser,+ evaluator: { select: { type: true } },},});const hasMore = versions.length > params.limit;@@ -547,7 +615,11 @@ export function createEvaluator(params: {},},},- include: { versions: latestVersion },+ include: {+ versions: latestVersionWithCreator,+ createdByUser,+ assignments: publicEvaluationRuleAssignments(params.input.projectId),+ },});}@@ -587,12 +659,17 @@ export function updateEvaluatorMetadata(params: {tx: PrismaTransaction;projectId: string;evaluatorId: string;- name: string;- description: string | null;+ name?: string;+ description?: string | null;}) {return params.tx.evaluator.update({where: { id: params.evaluatorId, projectId: params.projectId },- data: { name: params.name, description: params.description },+ data: {+ ...(params.name === undefined ? {} : { name: params.name }),+ ...(params.description === undefined+ ? {}+ : { description: params.description }),+ },});}diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorRouter.ts b/web/src/features/evals/v2/server/evaluators/evaluatorRouter.tsindex 5960449f24..9632a5afd1 100644--- a/web/src/features/evals/v2/server/evaluators/evaluatorRouter.ts+++ b/web/src/features/evals/v2/server/evaluators/evaluatorRouter.ts@@ -10,7 +10,7 @@ import {ActivationCostEstimatesSchema,CreateEvaluatorSchema,DeleteEvaluatorsSchema,- EvaluatorDefinitionSchema,+ EvaluatorDefinitionInputSchema,EvaluatorIdSchema,EvaluatorIdsSchema,EvaluatorOptionsSchema,@@ -26,7 +26,7 @@ import { getActivationCostEstimates } from "./activationCostService";const TestEvaluatorSchema = z.object({projectId: z.string(),evaluatorId: z.string(),- definition: EvaluatorDefinitionSchema,+ definition: EvaluatorDefinitionInputSchema,observationId: z.string(),traceId: z.string(),startTime: z.coerce.date(),diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorService.ts b/web/src/features/evals/v2/server/evaluators/evaluatorService.tsindex f2b64b7ca6..c6087723b1 100644--- a/web/src/features/evals/v2/server/evaluators/evaluatorService.ts+++ b/web/src/features/evals/v2/server/evaluators/evaluatorService.ts@@ -2,10 +2,12 @@ import { isDeepStrictEqual } from "node:util";import { z } from "zod";import {EvalTemplateType,+ type EvaluatorBlockReason,type FilterState,getBlockReasonForInvalidModelConfig,getCodeEvalVariableMapping,getEvaluatorBlockMetadata,+ isEvaluatorBlockReasonRecoverableByDefinitionUpdate,LangfuseConflictError,LangfuseNotFoundError,} from "@langfuse/shared";@@ -29,6 +31,7 @@ import {type EvaluatorDefinitionForPersistence,type EvaluatorListOrderBy,type EvaluatorVersionCursor,+ type PatchEvaluatorInput,type UpdateEvaluatorInput,encodeEvaluatorVersionCursor,EvaluatorDefinitionSchema,@@ -97,6 +100,17 @@ export class EvaluatorService {});}+ listCursor(params: {+ projectId: string;+ limit: number;+ cursor?: { createdAt: Date; id: string };+ }) {+ return repository.listEvaluatorsCursor({+ prisma: this.prisma,+ ...params,+ });+ }+listFilterOptions(projectId: string) {return repository.listEvaluatorFilterOptions({prisma: this.prisma,@@ -150,8 +164,8 @@ export class EvaluatorService {...params,cursor: params.cursor?.version,});- if (page.data.length === 0 && params.cursor === undefined) {- throw new LangfuseNotFoundError("Evaluator not found");+ if (page.data.length === 0) {+ await this.get(params.projectId, params.evaluatorId);}return {...page,@@ -259,6 +273,7 @@ export class EvaluatorService {}throw error;});+ await invalidateProjectEvalConfigCaches(input.projectId);await this.audit({action: "create",projectId: input.projectId,@@ -324,6 +339,7 @@ export class EvaluatorService {}),};});+ await invalidateProjectEvalConfigCaches(input.projectId);await this.audit({action: result.action,projectId: input.projectId,@@ -347,6 +363,7 @@ export class EvaluatorService {block,}),);+ await invalidateProjectEvalConfigCaches(input.projectId);await this.audit({action: "update",projectId: input.projectId,@@ -355,6 +372,47 @@ export class EvaluatorService {return evaluator;}+ async patch(input: PatchEvaluatorInput, createdByUserId: string | null) {+ const validationResult = input.definition+ ? await this.validatePatchedEvaluatorForPersistence(+ input,+ input.definition,+ )+ : null;+ const evaluator = await this.prisma.$transaction((tx) =>+ patchEvaluator({+ tx,+ input,+ createdByUserId,+ validationResult,+ }),+ );+ await invalidateProjectEvalConfigCaches(input.projectId);+ await this.audit({+ action: "update",+ projectId: input.projectId,+ evaluatorId: evaluator.id,+ });+ return evaluator;+ }++ private async validatePatchedEvaluatorForPersistence(+ input: PatchEvaluatorInput,+ definition: EvaluatorDefinition,+ ) {+ const current = await this.get(input.projectId, input.evaluatorId);+ return validateEvaluatorForPersistence({+ projectId: input.projectId,+ evaluatorId: input.evaluatorId,+ name: input.name ?? current.name,+ description:+ input.description === undefined+ ? current.description+ : input.description,+ definition,+ });+ }+async reactivate(params: { projectId: string; evaluatorId: string }) {const { projectId, evaluatorId } = params;const evaluator = await repository.findEvaluator({@@ -561,6 +619,70 @@ async function deleteEvaluator(params: {if (!deleted) throw new LangfuseNotFoundError("Evaluator not found");}+async function patchEvaluator(params: {+ tx: Prisma.TransactionClient;+ input: Omit<PatchEvaluatorInput, "definition"> & {+ definition?: EvaluatorDefinition;+ };+ createdByUserId: string | null;+ validationResult: Awaited<ReturnType<typeof validateEvaluatorForPersistence>>;+}) {+ const { tx, input, createdByUserId } = params;+ const current = await repository.findEvaluator({+ prisma: tx,+ projectId: input.projectId,+ evaluatorId: input.evaluatorId,+ });+ if (!current) throw new LangfuseNotFoundError("Evaluator not found");+ if (input.definition && current.type !== input.definition.type) {+ throw new LangfuseConflictError("Evaluator type cannot be changed");+ }++ if (input.name !== undefined || input.description !== undefined) {+ await repository.updateEvaluatorMetadata({+ tx,+ projectId: input.projectId,+ evaluatorId: input.evaluatorId,+ name: input.name,+ description: input.description,+ });+ }++ if (input.definition) {+ const latest = current.versions[0];+ if (!latest) throw new LangfuseNotFoundError("Evaluator version not found");+ const definitionChanged = !isDeepStrictEqual(+ toEvaluatorDefinition(current.type, latest),+ input.definition,+ );+ if (definitionChanged) {+ await repository.appendEvaluatorVersion({+ tx,+ evaluatorId: input.evaluatorId,+ version: latest.version + 1,+ definition: prepareEvaluatorDefinitionForPersistence(input.definition),+ createdByUserId,+ });+ }+ await reconcileEvaluatorBlock({+ tx,+ projectId: input.projectId,+ evaluatorId: input.evaluatorId,+ validationResult: params.validationResult,+ existingBlockedAt: current.blockedAt,+ existingBlockReason: current.blockReason,+ });+ }++ const updated = await repository.findEvaluator({+ prisma: tx,+ projectId: input.projectId,+ evaluatorId: input.evaluatorId,+ });+ if (!updated) throw new LangfuseNotFoundError("Evaluator not found");+ return updated;+}+async function updateEvaluator(params: {tx: Prisma.TransactionClient;input: UpdateEvaluatorInput;@@ -606,14 +728,14 @@ async function updateEvaluator(params: {});}- if (params.block) {- await repository.blockEvaluator({- tx,- projectId: input.projectId,- evaluatorId: input.evaluatorId,- ...params.block,- });- }+ await reconcileEvaluatorBlock({+ tx,+ projectId: input.projectId,+ evaluatorId: input.evaluatorId,+ validationResult: params.block,+ existingBlockedAt: current.blockedAt,+ existingBlockReason: current.blockReason,+ });const updated = await repository.findEvaluator({prisma: tx,@@ -624,6 +746,34 @@ async function updateEvaluator(params: {return updated;}+async function reconcileEvaluatorBlock(params: {+ tx: Prisma.TransactionClient;+ projectId: string;+ evaluatorId: string;+ validationResult: Awaited<ReturnType<typeof validateEvaluatorForPersistence>>;+ existingBlockedAt: Date | null;+ existingBlockReason: EvaluatorBlockReason | null;+}) {+ if (params.validationResult) {+ await repository.blockEvaluator({+ tx: params.tx,+ projectId: params.projectId,+ evaluatorId: params.evaluatorId,+ ...params.validationResult,+ });+ return;+ }+ if (!params.existingBlockedAt) return;+ if (+ !isEvaluatorBlockReasonRecoverableByDefinitionUpdate(+ params.existingBlockReason,+ )+ ) {+ return;+ }+ await repository.unblockEvaluator(params);+}+async function validateEvaluatorForPersistence(input: CreateEvaluatorInput | UpdateEvaluatorInput,) {diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorTypes.ts b/web/src/features/evals/v2/server/evaluators/evaluatorTypes.tsindex 1f0a402b69..3e80c4f9bc 100644--- a/web/src/features/evals/v2/server/evaluators/evaluatorTypes.ts+++ b/web/src/features/evals/v2/server/evaluators/evaluatorTypes.ts@@ -1,5 +1,7 @@import {+ EvalOutputDefinitionSchema,EvalTemplateType,+ extractVariables,EvaluatorSourceCodeLanguage,InvalidRequestError,PersistedEvalOutputDefinitionSchema,@@ -68,16 +70,50 @@ export const EvaluatorDefinitionSchema = z.discriminatedUnion("type", [CodeEvaluatorDefinitionSchema,]);+export const EvaluatorModelConfigSchema = z.object({+ provider: z.string().min(1),+ model: z.string().min(1),+ modelParams: ZodModelConfig.nullable().optional(),+});++const LlmEvaluatorDefinitionInputSchema = EvaluatorVersionBaseSchema.extend({+ type: z.literal(EvalTemplateType.LLM_AS_JUDGE),+ prompt: z.string().min(1),+ modelConfig: EvaluatorModelConfigSchema.nullable(),+ outputDefinition: EvalOutputDefinitionSchema,+});++export const EvaluatorDefinitionInputSchema = z+ .discriminatedUnion("type", [+ LlmEvaluatorDefinitionInputSchema,+ CodeEvaluatorDefinitionSchema,+ ])+ .transform(+ (definition): z.infer<typeof EvaluatorDefinitionSchema> =>+ definition.type === EvalTemplateType.CODE+ ? definition+ : {+ type: EvalTemplateType.LLM_AS_JUDGE,+ prompt: definition.prompt,+ provider: definition.modelConfig?.provider ?? null,+ model: definition.modelConfig?.model ?? null,+ modelParams: definition.modelConfig?.modelParams ?? null,+ vars: extractVariables(definition.prompt),+ variableMapping: definition.variableMapping,+ outputDefinition: definition.outputDefinition,+ },+ );+export const CreateEvaluatorSchema = EvaluatorMetadataSchema.extend({projectId: z.string(),evaluatorId: z.uuid().optional(),- definition: EvaluatorDefinitionSchema,+ definition: EvaluatorDefinitionInputSchema,});export const UpdateEvaluatorSchema = EvaluatorMetadataSchema.extend({projectId: z.string(),evaluatorId: z.string(),- definition: EvaluatorDefinitionSchema,+ definition: EvaluatorDefinitionInputSchema,});export const EvaluatorIdSchema = z.object({@@ -194,6 +230,11 @@ export type EvaluatorDefinitionForPersistence =});export type CreateEvaluatorInput = z.infer<typeof CreateEvaluatorSchema>;export type UpdateEvaluatorInput = z.infer<typeof UpdateEvaluatorSchema>;+export type PatchEvaluatorInput = Pick<+ UpdateEvaluatorInput,+ "projectId" | "evaluatorId"+> &+ Partial<Pick<UpdateEvaluatorInput, "name" | "description" | "definition">>;export type DeleteEvaluatorsInput = z.infer<typeof DeleteEvaluatorsSchema>;export type EvaluatorListOrderBy = z.infer<typeof ListEvaluatorsSchemadiff --git a/web/src/features/evals/v2/server/rules/ruleErrors.ts b/web/src/features/evals/v2/server/rules/ruleErrors.tsindex 7903c8986f..35c8ce62a3 100644--- a/web/src/features/evals/v2/server/rules/ruleErrors.ts+++ b/web/src/features/evals/v2/server/rules/ruleErrors.ts@@ -1,10 +1,29 @@-import { LangfuseConflictError, EvalTargetObject } from "@langfuse/shared";+import {+ InvalidRequestError,+ LangfuseConflictError,+ EvalTargetObject,+} from "@langfuse/shared";import {type PrismaClient,type Prisma,JobConfigState,} from "@langfuse/shared/src/db";+/**+ * Enforced for every surface (tRPC, MCP, public API) through this shared guard,+ * so the documented contract cannot drift per entry point.+ */+export function assertEnabledRuleHasAssignments(params: {+ enabled: boolean;+ assignmentCount: number;+}) {+ if (params.enabled && params.assignmentCount === 0) {+ throw new InvalidRequestError(+ "An enabled evaluation rule requires at least one evaluator assignment",+ );+ }+}+/*** Cap on simultaneously active evaluation rules per project. Enforced for every* surface (tRPC, MCP, public API) through `assertActiveRuleLimitNotExceeded`, sodiff --git a/web/src/features/evals/v2/server/rules/ruleRepository.ts b/web/src/features/evals/v2/server/rules/ruleRepository.tsindex dbec658a76..e94611d8f1 100644--- a/web/src/features/evals/v2/server/rules/ruleRepository.ts+++ b/web/src/features/evals/v2/server/rules/ruleRepository.ts@@ -50,7 +50,7 @@ const assignmentInclude = {};const ruleInclude = {- createdByUser: { select: { name: true, email: true } },+ createdByUser: { select: { id: true, name: true, email: true } },assignments: assignmentInclude,} satisfies Prisma.EvaluationRuleInclude;@@ -153,6 +153,45 @@ export async function listRules(params: {return { rules, totalItems };}+export async function listRulesCursor(params: {+ prisma: PrismaClient;+ input: Omit<ListRulesInput, "page"> & {+ cursor?: { createdAt: Date; id: string };+ };+}) {+ const baseWhere = ruleWhere(params.input);+ const where: Prisma.EvaluationRuleWhereInput = params.input.cursor+ ? {+ AND: [+ baseWhere,+ {+ OR: [+ { createdAt: { lt: params.input.cursor.createdAt } },+ {+ createdAt: params.input.cursor.createdAt,+ id: { lt: params.input.cursor.id },+ },+ ],+ },+ ],+ }+ : baseWhere;+ const records = await params.prisma.evaluationRule.findMany({+ where,+ orderBy: [{ createdAt: "desc" }, { id: "desc" }],+ take: params.input.limit + 1,+ include: ruleInclude,+ });+ const hasMore = records.length > params.input.limit;+ const rules = records.slice(0, params.input.limit);+ const last = rules.at(-1);+ return {+ rules,+ nextCursor:+ hasMore && last ? { createdAt: last.createdAt, id: last.id } : undefined,+ };+}+export async function listReusableFilterCandidates(params: {prisma: PrismaClient;projectId: string;@@ -195,7 +234,7 @@ export async function listRuleFilterOptions(params: {where: { projectId: params.projectId },select: {name: true,- createdByUser: { select: { name: true, email: true } },+ createdByUser: { select: { id: true, name: true, email: true } },},}),params.prisma.evaluationRule.findFirst({@@ -349,15 +388,34 @@ export async function deleteRule(params: {projectId: string;ruleId: string;}) {+ const result = await params.prisma.evaluationRule.deleteMany({+ where: { id: params.ruleId, projectId: params.projectId },+ });+ if (result.count === 0) return false;+// Executions carry no foreign key to the rule, so they have to go explicitly// or they keep showing up in the eval log of a deleted rule.await params.prisma.jobExecution.deleteMany({where: { projectId: params.projectId, jobConfigurationId: params.ruleId },});+ return true;+}++export async function deleteRules(params: {+ prisma: RulePrisma;+ projectId: string;+ ruleIds: string[];+}) {const result = await params.prisma.evaluationRule.deleteMany({- where: { id: params.ruleId, projectId: params.projectId },+ where: { projectId: params.projectId, id: { in: params.ruleIds } },});- return result.count > 0;+ await params.prisma.jobExecution.deleteMany({+ where: {+ projectId: params.projectId,+ jobConfigurationId: { in: params.ruleIds },+ },+ });+ return result.count;}export async function listSelectedRuleIds(params: {diff --git a/web/src/features/evals/v2/server/rules/ruleService.ts b/web/src/features/evals/v2/server/rules/ruleService.tsindex 9dc0912003..253b95f374 100644--- a/web/src/features/evals/v2/server/rules/ruleService.ts+++ b/web/src/features/evals/v2/server/rules/ruleService.ts@@ -35,7 +35,10 @@ import type {UpdateRuleInput,} from "./ruleTypes";import * as evaluatorRepository from "../evaluators/evaluatorRepository";-import { assertActiveRuleLimitNotExceeded } from "./ruleErrors";+import {+ assertActiveRuleLimitNotExceeded,+ assertEnabledRuleHasAssignments,+} from "./ruleErrors";import * as repository from "./ruleRepository";import { isLegacyEvalTarget } from "@/src/features/evals/utils/typeHelpers";import { prepareModernRuleVariableMapping } from "@/src/features/evals/v2/fns/variableMapping/prepareModernRuleVariableMapping";@@ -68,6 +71,18 @@ export class RuleService {return { rules: rules.map(toRuleResponse), totalItems };}+ async listCursor(+ input: Omit<ListRulesInput, "page"> & {+ cursor?: { createdAt: Date; id: string };+ },+ ) {+ const { rules, nextCursor } = await repository.listRulesCursor({+ prisma: this.prisma,+ input,+ });+ return { rules: rules.map(toRuleResponse), nextCursor };+ }+listFilterOptions(projectId: string) {return repository.listRuleFilterOptions({prisma: this.prisma,@@ -245,6 +260,10 @@ export class RuleService {normalized.filter,);this.assertUniqueAssignments(input.evaluatorAssignments);+ assertEnabledRuleHasAssignments({+ enabled: input.enabled,+ assignmentCount: input.evaluatorAssignments.length,+ });const rule = await this.prisma.$transaction(async (prisma) => {await assertActiveRuleLimitNotExceeded({prisma,@@ -356,6 +375,16 @@ export class RuleService {input.ruleId,);this.assertLegacyRuleUpdateAllowed(current.targetObject, input);+ const resultingAssignmentCount =+ input.evaluatorMappings?.length ?? current.assignments.length;+ assertEnabledRuleHasAssignments({+ enabled:+ input.enabled ??+ (input.evaluatorMappings?.length === 0+ ? false+ : current.status === JobConfigState.ACTIVE),+ assignmentCount: resultingAssignmentCount,+ });const currentTargetObject = current.targetObject as EvalTargetObject;const currentFilter = current.filter as FilterState;// Whether a rule targets experiments is derived from its filter, so an@@ -375,10 +404,9 @@ export class RuleService {targetObject: input.targetObject ?? inheritedTargetObject,filter: effectiveFilter,});- const filter = this.validateRuleFilters(- normalized.targetObject,- normalized.filter,- );+ const filter = isLegacyEvalTarget(currentTargetObject)+ ? currentFilter+ : this.validateRuleFilters(normalized.targetObject, normalized.filter);await assertActiveRuleLimitNotExceeded({prisma,projectId: input.projectId,@@ -441,6 +469,10 @@ export class RuleService {params.ruleId,);this.assertLegacyRuleCanBeEnabled(current.targetObject, params.enabled);+ assertEnabledRuleHasAssignments({+ enabled: params.enabled,+ assignmentCount: current.assignments.length,+ });await assertActiveRuleLimitNotExceeded({prisma,projectId: params.projectId,@@ -485,8 +517,10 @@ export class RuleService {async deleteMany(input: RuleSelectionInput) {const ruleIds = await this.prisma.$transaction(async (prisma) => {const ids = await repository.listSelectedRuleIds({ prisma, input });- await prisma.evaluationRule.deleteMany({- where: { projectId: input.projectId, id: { in: ids } },+ await repository.deleteRules({+ prisma,+ projectId: input.projectId,+ ruleIds: ids,});return ids;});@@ -512,6 +546,7 @@ export class RuleService {projectId: input.projectId,isBatchAction: true,search: input.search,+ filter: input.filter,};const ids = await repository.listSelectedRuleIds({prisma,@@ -530,6 +565,18 @@ export class RuleService {this.assertLegacyRuleCanBeEnabled("trace", input.enabled);}if (input.enabled) {+ const unassignedRule = await prisma.evaluationRule.findFirst({+ where: {+ projectId: input.projectId,+ id: { in: ids },+ assignments: { none: {} },+ },+ select: { id: true },+ });+ assertEnabledRuleHasAssignments({+ enabled: true,+ assignmentCount: unassignedRule ? 0 : 1,+ });// Only rules that are not already active consume a new slot.const newlyActivated = await prisma.evaluationRule.count({where: {diff --git a/web/src/features/evals/v2/server/rules/ruleTypes.ts b/web/src/features/evals/v2/server/rules/ruleTypes.tsindex a687913cd3..c11860b057 100644--- a/web/src/features/evals/v2/server/rules/ruleTypes.ts+++ b/web/src/features/evals/v2/server/rules/ruleTypes.ts@@ -70,7 +70,10 @@ export const CreateRuleSchema = RuleMetadataSchema.extend({projectId: z.string(),targetObject: z.enum([EvalTargetObject.EVENT, EvalTargetObject.EXPERIMENT])- .default(EvalTargetObject.EVENT),+ .default(EvalTargetObject.EVENT)+ .describe(+ "Deprecated: modern rules are event rules and experiment scope is expressed through filters.",+ ),enabled: z.boolean(),evaluatorAssignments: z.array(RuleAssignmentInputSchema).max(100),});@@ -115,7 +118,6 @@ const FilteredRuleSelectionSchema = z.object({projectId: z.string(),isBatchAction: z.literal(true),search: z.string().trim().max(200).optional(),- enabled: z.boolean().optional(),filter: ListRulesSchema.shape.filter,});diff --git a/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.clienttest.ts b/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.clienttest.tsindex 1d0b571d07..54a910dd75 100644--- a/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.clienttest.ts+++ b/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.clienttest.ts@@ -260,7 +260,6 @@ describe("createEvaluatorSetupStore", () => {},],outputDefinition: {- version: 2,dataType: "NUMERIC",score: {description: "Answer quality",diff --git a/web/src/features/mcp/core/run-mcp-tool.ts b/web/src/features/mcp/core/run-mcp-tool.tsindex d14d4d7c52..056b145e6d 100644--- a/web/src/features/mcp/core/run-mcp-tool.ts+++ b/web/src/features/mcp/core/run-mcp-tool.ts@@ -3,7 +3,7 @@ import { addUserToSpan, instrumentAsync } from "@langfuse/shared/src/server";import { SpanKind, type Span } from "@opentelemetry/api";import { ZodError } from "zod";-import { UnstablePublicApiError } from "@/src/features/public-api/server/unstable-public-api-error-contract";+import { StructuredPublicApiError } from "@/src/features/public-api";import { isUserInputError } from "./errors";import type { ServerContext } from "../types";@@ -66,7 +66,7 @@ export const runMcpTool = async <TResult>({if (error instanceof BaseError) {span.setAttribute("mcp.error.http_code", error.httpCode);}- if (error instanceof UnstablePublicApiError) {+ if (error instanceof StructuredPublicApiError) {span.setAttribute("mcp.error.code", error.code);}throw error;diff --git a/web/src/features/mcp/features/dashboardWidgets/tools/createDashboardWidget.ts b/web/src/features/mcp/features/dashboardWidgets/tools/createDashboardWidget.tsindex f8b454025b..ab0e359b04 100644--- a/web/src/features/mcp/features/dashboardWidgets/tools/createDashboardWidget.ts+++ b/web/src/features/mcp/features/dashboardWidgets/tools/createDashboardWidget.ts@@ -12,7 +12,7 @@ import {PostUnstableDashboardWidgetBody,PostUnstableDashboardWidgetView,} from "@/src/features/public-api/types/unstable-dashboard-widgets";-import { UnstablePublicApiError } from "@/src/features/public-api/server/unstable-public-api-error-contract";+import { StructuredPublicApiError } from "@/src/features/public-api";import { buildDashboardWidgetUrl } from "@langfuse/shared/src/server";export const DashboardWidgetFilterBaseSchema = z@@ -76,7 +76,7 @@ const throwActionableDashboardWidgetError = (error: unknown,input: z.infer<typeof PostUnstableDashboardWidgetBody>,): never => {- if (!(error instanceof UnstablePublicApiError)) throw error;+ if (!(error instanceof StructuredPublicApiError)) throw error;const field = error.details?.field;const allowedValues =diff --git a/web/src/features/mcp/features/evals/rule-schema.ts b/web/src/features/mcp/features/evals/rule-schema.tsindex b72cafeb0b..4d13fc117b 100644--- a/web/src/features/mcp/features/evals/rule-schema.ts+++ b/web/src/features/mcp/features/evals/rule-schema.ts@@ -9,8 +9,8 @@ import {UpdateRuleSchema,} from "@/src/features/evals/v2/server/rules/ruleTypes";import {- ObservationEvaluationRuleMapping,- PublicEvaluationRuleReadMapping,+ ObservationPromptVariableMappingInput,+ PromptVariableMappingRead,PublicEvaluatorType,} from "@/src/features/public-api/types/unstable-public-evals-contract";import { McpAdvancedFilterBaseSchema } from "../../core/filter-schema";@@ -31,7 +31,7 @@ const EvaluationRuleEvaluatorInputBase = {// The tool boundary speaks the public mapping contract (`variable`/`source`),// not the stored column ids. Handlers translate via `toStoredMappingList`.const VariableMappingSchema = z- .array(ObservationEvaluationRuleMapping)+ .array(ObservationPromptVariableMappingInput).describe("Optional rule-specific variable mapping for LLM evaluators. Omit for code evaluators, whose mapping is managed by Langfuse.",);@@ -159,7 +159,7 @@ export const EvaluationRuleResponseSchema = zevaluatorId: z.string(),evaluatorName: z.string(),evaluatorType: PublicEvaluatorType,- variableMapping: z.array(PublicEvaluationRuleReadMapping).nullable(),+ variableMapping: z.array(PromptVariableMappingRead).nullable(),}),),createdAt: z.date(),diff --git a/web/src/features/mcp/features/evals/tools/createEvaluator.ts b/web/src/features/mcp/features/evals/tools/createEvaluator.tsindex 063ea460bf..40ba78fdb4 100644--- a/web/src/features/mcp/features/evals/tools/createEvaluator.ts+++ b/web/src/features/mcp/features/evals/tools/createEvaluator.ts@@ -12,7 +12,7 @@ export const [createEvaluatorTool, handleCreateEvaluator] = defineTool({name: "createEvaluator",description: ["Create a new evaluator with a stable id. Names do not act as identity and may be reused.",- "Set type to `LLM_AS_JUDGE` and provide prompt + outputDefinition (provider, model, and modelParams are optional), or set type to `CODE` and provide sourceCode + sourceCodeLanguage.",+ "Set type to `LLM_AS_JUDGE` and provide prompt + outputDefinition. Omit modelConfig to use the project default, or provide modelConfig with provider, model, and optional modelParams. For `CODE`, provide sourceCode + sourceCodeLanguage.","Use updateEvaluator with the returned evaluatorId to update it or append a new immutable definition version.",].join(" "),baseSchema: McpEvaluatorInputBase,diff --git a/web/src/features/mcp/features/evals/tools/evaluatorInput.ts b/web/src/features/mcp/features/evals/tools/evaluatorInput.tsindex 1c47aa6cd0..af1472953a 100644--- a/web/src/features/mcp/features/evals/tools/evaluatorInput.ts+++ b/web/src/features/mcp/features/evals/tools/evaluatorInput.ts@@ -1,34 +1,46 @@-import { EvalTemplateType, extractVariables } from "@langfuse/shared";+import { EvalTemplateType, observationVariableMapping } from "@langfuse/shared";import { z } from "zod";import {CodeEvaluatorDefinitionSchema,CreateEvaluatorSchema,+ EvaluatorModelConfigSchema,LlmEvaluatorDefinitionSchema,} from "@/src/features/evals/v2/server/evaluators/evaluatorTypes";const CreateEvaluatorWithoutProjectSchema = CreateEvaluatorSchema.omit({projectId: true,});++const McpEvaluatorModelConfigSchema = EvaluatorModelConfigSchema.extend({+ modelParams: z+ .object({+ max_tokens: z.number().optional(),+ temperature: z.number().optional(),+ top_p: z.number().optional(),+ maxReasoningTokens: z.number().optional(),+ providerOptions: z.record(z.string(), z.any()).optional(),+ })+ .optional(),+});++const McpObservationVariableMappingSchema = observationVariableMapping.extend({+ jsonSelector: z.string().optional(),+});+export const McpEvaluatorInputBase = z.object({name: CreateEvaluatorSchema.shape.name,description: CreateEvaluatorSchema.shape.description.unwrap().optional(),type: z.enum(EvalTemplateType),prompt: LlmEvaluatorDefinitionSchema.shape.prompt.optional(),- provider: LlmEvaluatorDefinitionSchema.shape.provider.unwrap().optional(),- model: LlmEvaluatorDefinitionSchema.shape.model.unwrap().optional(),- modelParams: z.record(z.string(), z.unknown()).optional(),+ modelConfig: McpEvaluatorModelConfigSchema.optional().describe(+ "Optional custom model configuration. Omit to use the project default model.",+ ),outputDefinition: z.record(z.string(), z.unknown()).optional(),sourceCode: CodeEvaluatorDefinitionSchema.shape.sourceCode.optional(),sourceCodeLanguage:CodeEvaluatorDefinitionSchema.shape.sourceCodeLanguage.optional(),variableMapping: z- .array(- z.object({- templateVariable: z.string(),- selectedColumnId: z.string(),- jsonSelector: z.string().optional(),- }),- )+ .array(McpObservationVariableMappingSchema).optional().describe("Variable mappings for LLM-as-a-judge evaluators only."),});@@ -41,10 +53,7 @@ function toEvaluatorInput(input: z.infer<typeof McpEvaluatorInputBase>) {definition: {type: input.type,prompt: input.prompt!,- provider: input.provider ?? null,- model: input.model ?? null,- modelParams: input.modelParams ?? null,- vars: extractVariables(input.prompt!),+ modelConfig: input.modelConfig ?? null,variableMapping: input.variableMapping ?? null,outputDefinition: input.outputDefinition,},diff --git a/web/src/features/mcp/features/evals/tools/updateEvaluator.ts b/web/src/features/mcp/features/evals/tools/updateEvaluator.tsindex a658b11cbc..8de8f78d49 100644--- a/web/src/features/mcp/features/evals/tools/updateEvaluator.ts+++ b/web/src/features/mcp/features/evals/tools/updateEvaluator.ts@@ -12,7 +12,7 @@ export const [updateEvaluatorTool, handleUpdateEvaluator] = defineTool({name: "updateEvaluator",description: ["Update an evaluator by stable id. Definition changes append an immutable version; name and description changes do not.",- "Set type to `LLM_AS_JUDGE` and provide prompt + outputDefinition (provider, model, and modelParams are optional), or set type to `CODE` and provide sourceCode + sourceCodeLanguage.",+ "Set type to `LLM_AS_JUDGE` and provide prompt + outputDefinition. Omit modelConfig to use the project default, or provide modelConfig with provider, model, and optional modelParams. For `CODE`, provide sourceCode + sourceCodeLanguage.",].join(" "),baseSchema: McpUpdateEvaluatorInputBase,inputSchema: McpUpdateEvaluatorInput,diff --git a/web/src/features/public-api/index.ts b/web/src/features/public-api/index.tsnew file mode 100644index 0000000000..4f2daa4426--- /dev/null+++ b/web/src/features/public-api/index.ts@@ -0,0 +1,17 @@+export {+ DeleteEvaluatorResponse,+ Evaluator,+ ListEvaluatorVersionsResponse,+ ListEvaluatorsResponse,+ LlmAsJudgeEvaluator,+} from "./types/evaluation/evaluators";+export {+ DeleteEvaluationRuleResponse,+ EvaluationRule,+ ListEvaluationRulesResponse,+} from "./types/evaluation/evaluationRules";+export { PublicApiError } from "./types/public-api-errors";+export {+ createStructuredPublicApiError,+ StructuredPublicApiError,+} from "./types/structuredPublicApiError";diff --git a/web/src/features/public-api/server/RateLimitService.ts b/web/src/features/public-api/server/RateLimitService.tsindex ad14555f3e..bfbe86650b 100644--- a/web/src/features/public-api/server/RateLimitService.ts+++ b/web/src/features/public-api/server/RateLimitService.ts@@ -18,10 +18,10 @@ import {import { env as sharedEnv } from "@langfuse/shared/src/env";import { type NextApiResponse } from "next";import {- createUnstablePublicApiRateLimitError,- sendUnstablePublicApiErrorResponse,+ createStructuredPublicApiRateLimitError,+ sendStructuredPublicApiErrorResponse,type PublicApiErrorContract,-} from "@/src/features/public-api/server/unstable-public-api-error-contract";+} from "./structuredPublicApiErrorContract";import { type RateLimitUpgradePath } from "@/src/features/public-api/server/rateLimitUpgradePaths";export const RATE_LIMIT_REDIS_KEY_PREFIX = "rate-limit";@@ -217,9 +217,9 @@ export const sendRateLimitResponse = (res.setHeader(header, value);}- return sendUnstablePublicApiErrorResponse(+ return sendStructuredPublicApiErrorResponse(res,- createUnstablePublicApiRateLimitError(rateLimitRes, responseOptions),+ createStructuredPublicApiRateLimitError(rateLimitRes, responseOptions),);};diff --git a/web/src/features/public-api/server/createAuthedProjectAPIRoute.ts b/web/src/features/public-api/server/createAuthedProjectAPIRoute.tsindex 6af52dd881..5d1261932c 100644--- a/web/src/features/public-api/server/createAuthedProjectAPIRoute.ts+++ b/web/src/features/public-api/server/createAuthedProjectAPIRoute.ts@@ -23,12 +23,12 @@ import { env } from "@/src/env.mjs";import { isZodError } from "@/src/features/public-api/server/withMiddlewares";import { isPrismaException } from "@/src/utils/exceptions";import {- createUnstablePublicApiAuthError,- createUnstablePublicApiRequestValidationError,- sendUnstablePublicApiErrorResponse,- unstablePublicEvalsErrorContract,+ createStructuredPublicApiAuthError,+ createStructuredPublicApiRequestValidationError,+ sendStructuredPublicApiErrorResponse,+ structuredPublicApiErrorContract,type PublicApiErrorContract,-} from "@/src/features/public-api/server/unstable-public-api-error-contract";+} from "./structuredPublicApiErrorContract";import { clickHouseRouteForRequest } from "@/src/features/public-api/server/clickHouseRequestTags";import { attachDeprecation } from "@/src/features/public-api/server/deprecations";@@ -353,10 +353,10 @@ export const createAuthedProjectAPIRoute = <if (isPrismaException(error)) {traceException(error);- if (routeConfig.errorContract === unstablePublicEvalsErrorContract) {- return sendUnstablePublicApiErrorResponse(+ if (routeConfig.errorContract === structuredPublicApiErrorContract) {+ return sendStructuredPublicApiErrorResponse(res,- createUnstablePublicApiAuthError({+ createStructuredPublicApiAuthError({statusCode: 503,message: "Service Unavailable",}),@@ -370,10 +370,10 @@ export const createAuthedProjectAPIRoute = <const statusCode = error.status ?? 401;const message = error.message ?? "Authentication failed";- if (routeConfig.errorContract === unstablePublicEvalsErrorContract) {- return sendUnstablePublicApiErrorResponse(+ if (routeConfig.errorContract === structuredPublicApiErrorContract) {+ return sendStructuredPublicApiErrorResponse(res,- createUnstablePublicApiAuthError({ statusCode, message }),+ createStructuredPublicApiAuthError({ statusCode, message }),);}@@ -406,12 +406,12 @@ export const createAuthedProjectAPIRoute = <: ({} as z.infer<TQuery>);} catch (error) {if (- routeConfig.errorContract === unstablePublicEvalsErrorContract &&+ routeConfig.errorContract === structuredPublicApiErrorContract &&isZodError(error)) {- return sendUnstablePublicApiErrorResponse(+ return sendStructuredPublicApiErrorResponse(res,- createUnstablePublicApiRequestValidationError({+ createStructuredPublicApiRequestValidationError({error,requestPart: "query",}),@@ -428,12 +428,12 @@ export const createAuthedProjectAPIRoute = <: ({} as z.infer<TBody>);} catch (error) {if (- routeConfig.errorContract === unstablePublicEvalsErrorContract &&+ routeConfig.errorContract === structuredPublicApiErrorContract &&isZodError(error)) {- return sendUnstablePublicApiErrorResponse(+ return sendStructuredPublicApiErrorResponse(res,- createUnstablePublicApiRequestValidationError({+ createStructuredPublicApiRequestValidationError({error,requestPart: "body",}),diff --git a/web/src/features/public-api/server/evaluation/evaluationAdapters.ts b/web/src/features/public-api/server/evaluation/evaluationAdapters.tsnew file mode 100644index 0000000000..6dc59e7229--- /dev/null+++ b/web/src/features/public-api/server/evaluation/evaluationAdapters.ts@@ -0,0 +1,397 @@+import {+ EvalTemplateType,+ extractVariables,+ InternalServerError,+ observationVariableMappingList,+ PersistedEvalOutputDefinitionSchema,+ resolvePersistedEvalOutputDefinition,+ variableMappingList,+ type FilterState,+ type ObservationVariableMapping,+} from "@langfuse/shared";+import { logger } from "@langfuse/shared/src/server";+import {+ EvaluatorDefinitionInputSchema,+ isLegacyEvalTarget,+ type EvaluatorService,+ type RuleService,+} from "@/src/features/evals/server";+import {+ EvaluationRule,+ type CreateEvaluationRuleBodyType,+} from "@/src/features/public-api/types/evaluation/evaluationRules";+import {+ Evaluator,+ EvaluatorVersion,+ type EvaluatorDefinitionType,+} from "@/src/features/public-api/types/evaluation/evaluators";+import {+ LegacyPromptVariableMapping,+ PublicEvaluationRuleReadFilter,+ PUBLIC_EVALUATOR_TYPE_CODE,+ PUBLIC_EVALUATOR_TYPE_LLM_AS_JUDGE,+ type LegacyPromptVariableMappingType,+ type PromptVariableMappingInputType,+ type PromptVariableMappingReadType,+ type PublicEvaluatorOutputDefinitionType,+} from "@/src/features/public-api/types/evaluation/publicEvalsContract";++const PUBLIC_MAPPING_SOURCE_TO_INTERNAL_COLUMN: Record<+ PromptVariableMappingInputType["source"],+ ObservationVariableMapping["selectedColumnId"]+> = {+ input: "input",+ output: "output",+ metadata: "metadata",+ tool_calls: "toolCalls",+ expected_output: "experimentItemExpectedOutput",+ experiment_item_metadata: "experimentItemMetadata",+};++const INTERNAL_MAPPING_COLUMN_TO_PUBLIC_SOURCE: Record<+ string,+ PromptVariableMappingInputType["source"]+> = {+ input: "input",+ output: "output",+ metadata: "metadata",+ toolCalls: "tool_calls",+ expected_output: "expected_output",+ expectedOutput: "expected_output",+ experiment_item_expected_output: "expected_output",+ experimentItemExpectedOutput: "expected_output",+ experimentItemMetadata: "experiment_item_metadata",+ experiment_item_metadata: "experiment_item_metadata",+};++function toStoredMappingList(mappings: PromptVariableMappingInputType[]) {+ return observationVariableMappingList.parse(+ mappings.map((mapping) => ({+ templateVariable: mapping.variable,+ selectedColumnId:+ PUBLIC_MAPPING_SOURCE_TO_INTERNAL_COLUMN[mapping.source],+ jsonSelector: mapping.jsonPath ?? null,+ })),+ );+}++function toApiReadMappings(mappings: unknown): PromptVariableMappingReadType[] {+ const parsed = observationVariableMappingList.safeParse(mappings);+ if (!parsed.success) {+ logger.error("Failed to parse public evaluation rule mappings", {+ issues: parsed.error.issues,+ });+ throw new InternalServerError("Evaluation rule mapping is corrupted");+ }++ return parsed.data.map((mapping) => {+ const source =+ INTERNAL_MAPPING_COLUMN_TO_PUBLIC_SOURCE[mapping.selectedColumnId];+ if (!source && mapping.selectedColumnId.trim()) {+ throw new InternalServerError("Evaluation rule mapping is corrupted");+ }+ return {+ variable: mapping.templateVariable,+ source: source ?? null,+ ...(mapping.jsonSelector ? { jsonPath: mapping.jsonSelector } : {}),+ };+ });+}++function toApiLegacyMappings(+ mappings: unknown,+): LegacyPromptVariableMappingType[] {+ const parsed = variableMappingList.safeParse(mappings);+ if (!parsed.success) {+ logger.error("Failed to parse public legacy evaluation rule mappings", {+ issues: parsed.error.issues,+ });+ throw new InternalServerError("Evaluation rule mapping is corrupted");+ }++ return LegacyPromptVariableMapping.array().parse(+ parsed.data.map((mapping) => ({+ mappingType: "legacy",+ variable: mapping.templateVariable,+ langfuseObject: mapping.langfuseObject,+ objectName: mapping.objectName ?? null,+ source: mapping.selectedColumnId,+ ...(mapping.jsonSelector ? { jsonPath: mapping.jsonSelector } : {}),+ })),+ );+}++function toPublicOutputDefinition(outputDefinition: unknown) {+ const parsed =+ PersistedEvalOutputDefinitionSchema.safeParse(outputDefinition);+ if (!parsed.success) {+ logger.error("Failed to parse public evaluator output definition", {+ issues: parsed.error.issues,+ });+ throw new InternalServerError("Evaluator output definition is corrupted");+ }++ const resolved = resolvePersistedEvalOutputDefinition(parsed.data);+ const descriptions = {+ ...(resolved.reasoningDescription+ ? { scoreReasoningInstructions: resolved.reasoningDescription }+ : {}),+ ...(resolved.scoreDescription+ ? { scoreValueInstructions: resolved.scoreDescription }+ : {}),+ };+ if (resolved.dataType === "NUMERIC") {+ return {+ dataType: resolved.dataType,+ ...descriptions,+ ...(resolved.minValue === undefined+ ? {}+ : { minValue: resolved.minValue }),+ ...(resolved.maxValue === undefined+ ? {}+ : { maxValue: resolved.maxValue }),+ };+ }+ if (resolved.dataType === "BOOLEAN") {+ return { dataType: resolved.dataType, ...descriptions };+ }+ return {+ dataType: resolved.dataType,+ ...descriptions,+ categories: resolved.categories,+ shouldAllowMultipleMatches: resolved.shouldAllowMultipleMatches,+ };+}++function toCreator(+ creator: {+ id: string;+ name: string | null;+ } | null,+) {+ return creator ? { id: creator.id, name: creator.name } : null;+}++function toInternalOutputDefinition(+ outputDefinition: PublicEvaluatorOutputDefinitionType,+) {+ const descriptions = {+ reasoning: {+ description: outputDefinition.scoreReasoningInstructions ?? "",+ },+ scoreDescription: outputDefinition.scoreValueInstructions ?? "",+ };+ if (outputDefinition.dataType === "NUMERIC") {+ return {+ dataType: outputDefinition.dataType,+ reasoning: descriptions.reasoning,+ score: {+ description: descriptions.scoreDescription,+ ...(outputDefinition.minValue === undefined+ ? {}+ : { minValue: outputDefinition.minValue }),+ ...(outputDefinition.maxValue === undefined+ ? {}+ : { maxValue: outputDefinition.maxValue }),+ },+ };+ }+ if (outputDefinition.dataType === "BOOLEAN") {+ return {+ dataType: outputDefinition.dataType,+ reasoning: descriptions.reasoning,+ score: { description: descriptions.scoreDescription },+ };+ }+ return {+ dataType: outputDefinition.dataType,+ reasoning: descriptions.reasoning,+ score: {+ description: descriptions.scoreDescription,+ categories: outputDefinition.categories,+ shouldAllowMultipleMatches: outputDefinition.shouldAllowMultipleMatches,+ },+ };+}++export function toEvaluatorServiceDefinition(+ definition: EvaluatorDefinitionType,+) {+ return EvaluatorDefinitionInputSchema.parse(+ definition.type === PUBLIC_EVALUATOR_TYPE_CODE+ ? {+ type: EvalTemplateType.CODE,+ sourceCode: definition.sourceCode,+ sourceCodeLanguage: definition.sourceCodeLanguage,+ }+ : {+ type: EvalTemplateType.LLM_AS_JUDGE,+ prompt: definition.prompt[0]!.content,+ modelConfig: definition.modelConfig ?? null,+ variableMapping:+ definition.variableMapping == null+ ? null+ : toStoredMappingList(definition.variableMapping),+ outputDefinition: toInternalOutputDefinition(+ definition.outputDefinition,+ ),+ },+ );+}++type ServiceEvaluator = Awaited<ReturnType<EvaluatorService["get"]>>;+type ServiceEvaluatorVersion =+ | ServiceEvaluator["versions"][number]+ | Awaited<ReturnType<EvaluatorService["listVersions"]>>["data"][number];++export function toPublicEvaluatorVersion(+ evaluatorType: EvalTemplateType,+ version: ServiceEvaluatorVersion,+) {+ const common = {+ id: version.id,+ version: version.version,+ createdAt: version.createdAt,+ createdBy: toCreator(version.createdByUser),+ };++ if (evaluatorType === EvalTemplateType.CODE) {+ if (!version.sourceCode || !version.sourceCodeLanguage) {+ throw new InternalServerError("Code evaluator definition is corrupted");+ }+ return EvaluatorVersion.parse({+ ...common,+ type: PUBLIC_EVALUATOR_TYPE_CODE,+ sourceCode: version.sourceCode,+ sourceCodeLanguage: version.sourceCodeLanguage,+ });+ }++ if (!version.prompt) {+ throw new InternalServerError("Evaluator prompt is corrupted");+ }+ const prompt = [{ role: "user" as const, content: version.prompt }];+ return EvaluatorVersion.parse({+ ...common,+ type: PUBLIC_EVALUATOR_TYPE_LLM_AS_JUDGE,+ prompt,+ variables:+ version.vars.length > 0+ ? version.vars+ : extractVariables(prompt.map(({ content }) => content).join("\n")),+ variableMapping:+ version.variableMapping === null+ ? null+ : toApiReadMappings(version.variableMapping),+ modelConfig:+ version.provider && version.model+ ? {+ provider: version.provider,+ model: version.model,+ }+ : null,+ outputDefinition: toPublicOutputDefinition(version.outputDefinition),+ });+}++export function toPublicEvaluator(evaluator: ServiceEvaluator) {+ const latestVersion = evaluator.versions[0];+ if (!latestVersion) {+ throw new InternalServerError("Evaluator version is missing");+ }+ const common = {+ id: evaluator.id,+ name: evaluator.name,+ description: evaluator.description,+ createdBy: toCreator(evaluator.createdByUser),+ status: evaluator.blockedAt ? ("paused" as const) : ("active" as const),+ pausedAt: evaluator.blockedAt,+ pausedReason: evaluator.blockReason,+ pausedMessage: evaluator.blockMessage,+ evaluationRuleAssignments: evaluator.assignments.map(+ ({ evaluationRule, variableMapping }) => ({+ evaluationRuleId: evaluationRule.id,+ ...(variableMapping === null+ ? {}+ : {+ variableMappingOverride: isLegacyEvalTarget(+ evaluationRule.targetObject,+ )+ ? toApiLegacyMappings(variableMapping)+ : toApiReadMappings(variableMapping),+ }),+ }),+ ),+ createdAt: evaluator.createdAt,+ updatedAt: evaluator.updatedAt,+ };+ const {+ id: versionId,+ createdAt: versionCreatedAt,+ createdBy: versionCreatedBy,+ ...version+ } = toPublicEvaluatorVersion(evaluator.type, latestVersion);+ return Evaluator.parse({+ ...common,+ ...version,+ versionId,+ versionCreatedAt,+ versionCreatedBy,+ });+}++export function toInternalFilters(filters: FilterState) {+ return filters.map((filter) =>+ filter.column === "datasetId"+ ? { ...filter, column: "experimentDatasetId" }+ : filter,+ ) as FilterState;+}++function toPublicFilters(filters: unknown) {+ return PublicEvaluationRuleReadFilter.array()+ .parse(filters)+ .map((filter) =>+ filter.column === "experimentDatasetId"+ ? { ...filter, column: "datasetId" }+ : filter,+ );+}++export function toInternalAssignments(+ assignments: CreateEvaluationRuleBodyType["evaluatorAssignments"],+) {+ return assignments.map((assignment) => ({+ evaluatorId: assignment.evaluatorId,+ variableMapping:+ assignment.variableMapping === null+ ? null+ : (toStoredMappingList(+ assignment.variableMapping,+ ) as ObservationVariableMapping[]),+ }));+}++type ServiceRule = Awaited<ReturnType<RuleService["get"]>>;++export function toPublicRule(rule: ServiceRule) {+ return EvaluationRule.parse({+ id: rule.id,+ name: rule.name,+ createdBy: toCreator(rule.createdByUser),+ enabled: rule.enabled,+ sampling: rule.sampling,+ filter: toPublicFilters(rule.filter),+ evaluatorAssignments: rule.assignments.map((assignment) => ({+ evaluatorId: assignment.evaluator.id,+ variableMapping:+ assignment.variableMapping === null+ ? null+ : isLegacyEvalTarget(rule.targetObject)+ ? toApiLegacyMappings(assignment.variableMapping)+ : toApiReadMappings(assignment.variableMapping),+ })),+ createdAt: rule.createdAt,+ updatedAt: rule.updatedAt,+ });+}diff --git a/web/src/features/public-api/server/evaluation/evaluationRuleApiHandler.ts b/web/src/features/public-api/server/evaluation/evaluationRuleApiHandler.tsnew file mode 100644index 0000000000..f8a087d6e1--- /dev/null+++ b/web/src/features/public-api/server/evaluation/evaluationRuleApiHandler.ts@@ -0,0 +1,53 @@+import {+ createStablePublicApiRoute,+ withStablePublicApiMiddlewares,+} from "@/src/features/public-api/server/stablePublicApiRoute";+import {+ deleteEvaluationRuleForPublicApi,+ getEvaluationRuleForPublicApi,+ updateEvaluationRuleForPublicApi,+} from "./evaluationRuleApiService";+import {+ DeleteEvaluationRuleResponse,+ EvaluationRule,+ EvaluationRuleIdQuery,+ UpdateEvaluationRuleBody,+} from "@/src/features/public-api/types/evaluation/evaluationRules";++export const evaluationRuleApiHandler = withStablePublicApiMiddlewares({+ GET: createStablePublicApiRoute({+ name: "Get evaluation rule",+ querySchema: EvaluationRuleIdQuery,+ responseSchema: EvaluationRule,+ fn: ({ query, auth }) =>+ getEvaluationRuleForPublicApi({+ projectId: auth.scope.projectId,+ evaluationRuleId: query.evaluationRuleId,+ auditScope: auth.scope,+ }),+ }),+ PATCH: createStablePublicApiRoute({+ name: "Update evaluation rule",+ querySchema: EvaluationRuleIdQuery,+ bodySchema: UpdateEvaluationRuleBody,+ responseSchema: EvaluationRule,+ fn: ({ query, body, auth }) =>+ updateEvaluationRuleForPublicApi({+ projectId: auth.scope.projectId,+ evaluationRuleId: query.evaluationRuleId,+ input: body,+ auditScope: auth.scope,+ }),+ }),+ DELETE: createStablePublicApiRoute({+ name: "Delete evaluation rule",+ querySchema: EvaluationRuleIdQuery,+ responseSchema: DeleteEvaluationRuleResponse,+ fn: ({ query, auth }) =>+ deleteEvaluationRuleForPublicApi({+ projectId: auth.scope.projectId,+ evaluationRuleId: query.evaluationRuleId,+ auditScope: auth.scope,+ }),+ }),+});diff --git a/web/src/features/public-api/server/evaluation/evaluationRuleApiService.ts b/web/src/features/public-api/server/evaluation/evaluationRuleApiService.tsnew file mode 100644index 0000000000..8ca5ae0043--- /dev/null+++ b/web/src/features/public-api/server/evaluation/evaluationRuleApiService.ts@@ -0,0 +1,135 @@+import { EvalTargetObject } from "@langfuse/shared";+import { prisma } from "@langfuse/shared/src/db";+import type { ApiAccessScope } from "@langfuse/shared/src/server";+import { auditLog } from "@/src/features/audit-logs/server";+import {+ JOB_CONFIGURATION_AUDIT_LOG_RESOURCE_TYPE,+ RuleService,+} from "@/src/features/evals/server";+import {+ type CreateEvaluationRuleBodyType,+ type UpdateEvaluationRuleBodyType,+} from "@/src/features/public-api/types/evaluation/evaluationRules";+import {+ encodeResourceCursor,+ type ResourceCursorType,+} from "@/src/features/public-api/types/evaluation/evaluators";+import {+ toInternalAssignments,+ toInternalFilters,+ toPublicRule,+} from "./evaluationAdapters";++function ruleService(auditScope: ApiAccessScope) {+ return new RuleService(prisma, ({ action, ruleId, projectId }) =>+ auditLog({+ action,+ resourceType: JOB_CONFIGURATION_AUDIT_LOG_RESOURCE_TYPE,+ resourceId: ruleId,+ projectId,+ orgId: auditScope.orgId,+ apiKeyId: auditScope.apiKeyId,+ }),+ );+}++export async function listEvaluationRulesForPublicApi(params: {+ projectId: string;+ limit: number;+ cursor?: ResourceCursorType;+ auditScope: ApiAccessScope;+}) {+ const result = await ruleService(params.auditScope).listCursor({+ projectId: params.projectId,+ limit: params.limit,+ cursor: params.cursor+ ? {+ createdAt: new Date(params.cursor.lastCreatedAt),+ id: params.cursor.lastId,+ }+ : undefined,+ });+ return {+ data: result.rules.map(toPublicRule),+ meta: result.nextCursor+ ? {+ cursor: encodeResourceCursor({+ v: 1,+ lastCreatedAt: result.nextCursor.createdAt.toISOString(),+ lastId: result.nextCursor.id,+ }),+ }+ : {},+ };+}++export async function getEvaluationRuleForPublicApi(params: {+ projectId: string;+ evaluationRuleId: string;+ auditScope: ApiAccessScope;+}) {+ return toPublicRule(+ await ruleService(params.auditScope).get(+ params.projectId,+ params.evaluationRuleId,+ ),+ );+}++export async function createEvaluationRuleForPublicApi(params: {+ projectId: string;+ input: CreateEvaluationRuleBodyType;+ auditScope: ApiAccessScope;+}) {+ return toPublicRule(+ await ruleService(params.auditScope).create(+ {+ projectId: params.projectId,+ name: params.input.name,+ targetObject: EvalTargetObject.EVENT,+ enabled: params.input.enabled,+ sampling: params.input.sampling,+ filter: toInternalFilters(params.input.filter),+ evaluatorAssignments: toInternalAssignments(+ params.input.evaluatorAssignments,+ ),+ },+ null,+ ),+ );+}++export async function updateEvaluationRuleForPublicApi(params: {+ projectId: string;+ evaluationRuleId: string;+ input: UpdateEvaluationRuleBodyType;+ auditScope: ApiAccessScope;+}) {+ return toPublicRule(+ await ruleService(params.auditScope).update({+ projectId: params.projectId,+ ruleId: params.evaluationRuleId,+ name: params.input.name,+ enabled: params.input.enabled,+ sampling: params.input.sampling,+ filter: params.input.filter+ ? toInternalFilters(params.input.filter)+ : undefined,+ evaluatorMappings: params.input.evaluatorAssignments+ ? toInternalAssignments(params.input.evaluatorAssignments)+ : undefined,+ }),+ );+}++export async function deleteEvaluationRuleForPublicApi(params: {+ projectId: string;+ evaluationRuleId: string;+ auditScope: ApiAccessScope;+}) {+ await ruleService(params.auditScope).delete(+ params.projectId,+ params.evaluationRuleId,+ );+ return { id: params.evaluationRuleId };+}diff --git a/web/src/features/public-api/server/evaluation/evaluationRulesApiHandler.ts b/web/src/features/public-api/server/evaluation/evaluationRulesApiHandler.tsnew file mode 100644index 0000000000..ec8910e300--- /dev/null+++ b/web/src/features/public-api/server/evaluation/evaluationRulesApiHandler.ts@@ -0,0 +1,41 @@+import {+ createStablePublicApiRoute,+ withStablePublicApiMiddlewares,+} from "@/src/features/public-api/server/stablePublicApiRoute";+import {+ createEvaluationRuleForPublicApi,+ listEvaluationRulesForPublicApi,+} from "./evaluationRuleApiService";+import {+ CreateEvaluationRuleBody,+ EvaluationRule,+ ListEvaluationRulesQuery,+ ListEvaluationRulesResponse,+} from "@/src/features/public-api/types/evaluation/evaluationRules";++export const evaluationRulesApiHandler = withStablePublicApiMiddlewares({+ GET: createStablePublicApiRoute({+ name: "List evaluation rules",+ querySchema: ListEvaluationRulesQuery,+ responseSchema: ListEvaluationRulesResponse,+ fn: ({ query, auth }) =>+ listEvaluationRulesForPublicApi({+ projectId: auth.scope.projectId,+ limit: query.limit,+ cursor: query.cursor,+ auditScope: auth.scope,+ }),+ }),+ POST: createStablePublicApiRoute({+ name: "Create evaluation rule",+ bodySchema: CreateEvaluationRuleBody,+ responseSchema: EvaluationRule,+ successStatusCode: 201,+ fn: ({ body, auth }) =>+ createEvaluationRuleForPublicApi({+ projectId: auth.scope.projectId,+ input: body,+ auditScope: auth.scope,+ }),+ }),+});diff --git a/web/src/features/public-api/server/evaluation/evaluatorApiHandler.ts b/web/src/features/public-api/server/evaluation/evaluatorApiHandler.tsnew file mode 100644index 0000000000..310b204b6e--- /dev/null+++ b/web/src/features/public-api/server/evaluation/evaluatorApiHandler.ts@@ -0,0 +1,53 @@+import {+ createStablePublicApiRoute,+ withStablePublicApiMiddlewares,+} from "@/src/features/public-api/server/stablePublicApiRoute";+import {+ deleteEvaluatorForPublicApi,+ getEvaluatorForPublicApi,+ updateEvaluatorForPublicApi,+} from "./evaluatorApiService";+import {+ DeleteEvaluatorResponse,+ Evaluator,+ EvaluatorIdQuery,+ UpdateEvaluatorBody,+} from "@/src/features/public-api/types/evaluation/evaluators";++export const evaluatorApiHandler = withStablePublicApiMiddlewares({+ GET: createStablePublicApiRoute({+ name: "Get evaluator",+ querySchema: EvaluatorIdQuery,+ responseSchema: Evaluator,+ fn: ({ query, auth }) =>+ getEvaluatorForPublicApi({+ projectId: auth.scope.projectId,+ evaluatorId: query.evaluatorId,+ auditScope: auth.scope,+ }),+ }),+ PATCH: createStablePublicApiRoute({+ name: "Update evaluator",+ querySchema: EvaluatorIdQuery,+ bodySchema: UpdateEvaluatorBody,+ responseSchema: Evaluator,+ fn: ({ query, body, auth }) =>+ updateEvaluatorForPublicApi({+ projectId: auth.scope.projectId,+ evaluatorId: query.evaluatorId,+ input: body,+ auditScope: auth.scope,+ }),+ }),+ DELETE: createStablePublicApiRoute({+ name: "Delete evaluator",+ querySchema: EvaluatorIdQuery,+ responseSchema: DeleteEvaluatorResponse,+ fn: ({ query, auth }) =>+ deleteEvaluatorForPublicApi({+ projectId: auth.scope.projectId,+ evaluatorId: query.evaluatorId,+ auditScope: auth.scope,+ }),+ }),+});diff --git a/web/src/features/public-api/server/evaluation/evaluatorApiService.ts b/web/src/features/public-api/server/evaluation/evaluatorApiService.tsnew file mode 100644index 0000000000..a319e4836f--- /dev/null+++ b/web/src/features/public-api/server/evaluation/evaluatorApiService.ts@@ -0,0 +1,170 @@+import { EvalTemplateType, ZodModelConfig } from "@langfuse/shared";+import { prisma } from "@langfuse/shared/src/db";+import type { ApiAccessScope } from "@langfuse/shared/src/server";+import { auditLog } from "@/src/features/audit-logs/server";+import {+ EVAL_TEMPLATE_AUDIT_LOG_RESOURCE_TYPE,+ EvaluatorService,+} from "@/src/features/evals/server";+import {+ encodeResourceCursor,+ type CreateEvaluatorBodyType,+ type ResourceCursorType,+ type UpdateEvaluatorBodyType,+} from "@/src/features/public-api/types/evaluation/evaluators";+import {+ toEvaluatorServiceDefinition,+ toPublicEvaluator,+ toPublicEvaluatorVersion,+} from "./evaluationAdapters";++function evaluatorService(auditScope: ApiAccessScope) {+ return new EvaluatorService(prisma, ({ action, evaluatorId, projectId }) =>+ auditLog({+ action,+ resourceType: EVAL_TEMPLATE_AUDIT_LOG_RESOURCE_TYPE,+ resourceId: evaluatorId,+ projectId,+ orgId: auditScope.orgId,+ apiKeyId: auditScope.apiKeyId,+ }),+ );+}++export async function listEvaluatorsForPublicApi(params: {+ projectId: string;+ limit: number;+ cursor?: ResourceCursorType;+ auditScope: ApiAccessScope;+}) {+ const result = await evaluatorService(params.auditScope).listCursor({+ projectId: params.projectId,+ limit: params.limit,+ cursor: params.cursor+ ? {+ createdAt: new Date(params.cursor.lastCreatedAt),+ id: params.cursor.lastId,+ }+ : undefined,+ });+ return {+ data: result.evaluators.map(toPublicEvaluator),+ meta: result.nextCursor+ ? {+ cursor: encodeResourceCursor({+ v: 1,+ lastCreatedAt: result.nextCursor.createdAt.toISOString(),+ lastId: result.nextCursor.id,+ }),+ }+ : {},+ };+}++export async function getEvaluatorForPublicApi(params: {+ projectId: string;+ evaluatorId: string;+ auditScope: ApiAccessScope;+}) {+ return toPublicEvaluator(+ await evaluatorService(params.auditScope).get(+ params.projectId,+ params.evaluatorId,+ ),+ );+}++export async function createEvaluatorForPublicApi(params: {+ projectId: string;+ input: CreateEvaluatorBodyType;+ auditScope: ApiAccessScope;+}) {+ return toPublicEvaluator(+ await evaluatorService(params.auditScope).create(+ {+ projectId: params.projectId,+ name: params.input.name,+ description: params.input.description ?? null,+ definition: toEvaluatorServiceDefinition(params.input),+ },+ null,+ ),+ );+}++export async function updateEvaluatorForPublicApi(params: {+ projectId: string;+ evaluatorId: string;+ input: UpdateEvaluatorBodyType;+ auditScope: ApiAccessScope;+}) {+ const service = evaluatorService(params.auditScope);+ let definition =+ "type" in params.input+ ? toEvaluatorServiceDefinition(params.input)+ : undefined;++ if (+ definition?.type === EvalTemplateType.LLM_AS_JUDGE &&+ "type" in params.input &&+ params.input.type === "llm_as_judge" &&+ params.input.modelConfig === undefined+ ) {+ const current = await service.get(params.projectId, params.evaluatorId);+ const latestVersion = current.versions[0];+ if (current.type === EvalTemplateType.LLM_AS_JUDGE && latestVersion) {+ definition = {+ ...definition,+ provider: latestVersion.provider,+ model: latestVersion.model,+ modelParams: ZodModelConfig.nullable().parse(latestVersion.modelParams),+ };+ }+ }++ return toPublicEvaluator(+ await service.patch(+ {+ projectId: params.projectId,+ evaluatorId: params.evaluatorId,+ name: params.input.name,+ description: params.input.description,+ definition,+ },+ null,+ ),+ );+}++export async function deleteEvaluatorForPublicApi(params: {+ projectId: string;+ evaluatorId: string;+ auditScope: ApiAccessScope;+}) {+ await evaluatorService(params.auditScope).delete(+ params.projectId,+ params.evaluatorId,+ );+ return { id: params.evaluatorId };+}++export async function listEvaluatorVersionsForPublicApi(params: {+ projectId: string;+ evaluatorId: string;+ limit: number;+ cursor?: { v: 1; version: number };+ auditScope: ApiAccessScope;+}) {+ const result = await evaluatorService(params.auditScope).listVersions({+ projectId: params.projectId,+ evaluatorId: params.evaluatorId,+ limit: params.limit,+ cursor: params.cursor,+ });+ return {+ data: result.data.map((version) =>+ toPublicEvaluatorVersion(version.evaluator.type, version),+ ),+ meta: result.nextCursor ? { cursor: result.nextCursor } : {},+ };+}diff --git a/web/src/features/public-api/server/evaluation/evaluatorVersionsApiHandler.ts b/web/src/features/public-api/server/evaluation/evaluatorVersionsApiHandler.tsnew file mode 100644index 0000000000..77650d8acc--- /dev/null+++ b/web/src/features/public-api/server/evaluation/evaluatorVersionsApiHandler.ts@@ -0,0 +1,25 @@+import {+ createStablePublicApiRoute,+ withStablePublicApiMiddlewares,+} from "@/src/features/public-api/server/stablePublicApiRoute";+import { listEvaluatorVersionsForPublicApi } from "./evaluatorApiService";+import {+ ListEvaluatorVersionsQuery,+ ListEvaluatorVersionsResponse,+} from "@/src/features/public-api/types/evaluation/evaluators";++export const evaluatorVersionsApiHandler = withStablePublicApiMiddlewares({+ GET: createStablePublicApiRoute({+ name: "List evaluator versions",+ querySchema: ListEvaluatorVersionsQuery,+ responseSchema: ListEvaluatorVersionsResponse,+ fn: ({ query, auth }) =>+ listEvaluatorVersionsForPublicApi({+ projectId: auth.scope.projectId,+ evaluatorId: query.evaluatorId,+ limit: query.limit,+ cursor: query.cursor,+ auditScope: auth.scope,+ }),+ }),+});diff --git a/web/src/features/public-api/server/evaluation/evaluatorsApiHandler.ts b/web/src/features/public-api/server/evaluation/evaluatorsApiHandler.tsnew file mode 100644index 0000000000..5a6cceac47--- /dev/null+++ b/web/src/features/public-api/server/evaluation/evaluatorsApiHandler.ts@@ -0,0 +1,41 @@+import {+ createStablePublicApiRoute,+ withStablePublicApiMiddlewares,+} from "@/src/features/public-api/server/stablePublicApiRoute";+import {+ createEvaluatorForPublicApi,+ listEvaluatorsForPublicApi,+} from "./evaluatorApiService";+import {+ CreateEvaluatorBody,+ Evaluator,+ ListEvaluatorsQuery,+ ListEvaluatorsResponse,+} from "@/src/features/public-api/types/evaluation/evaluators";++export const evaluatorsApiHandler = withStablePublicApiMiddlewares({+ GET: createStablePublicApiRoute({+ name: "List evaluators",+ querySchema: ListEvaluatorsQuery,+ responseSchema: ListEvaluatorsResponse,+ fn: ({ query, auth }) =>+ listEvaluatorsForPublicApi({+ projectId: auth.scope.projectId,+ limit: query.limit,+ cursor: query.cursor,+ auditScope: auth.scope,+ }),+ }),+ POST: createStablePublicApiRoute({+ name: "Create evaluator",+ bodySchema: CreateEvaluatorBody,+ responseSchema: Evaluator,+ successStatusCode: 201,+ fn: ({ body, auth }) =>+ createEvaluatorForPublicApi({+ projectId: auth.scope.projectId,+ input: body,+ auditScope: auth.scope,+ }),+ }),+});diff --git a/web/src/features/public-api/server/index.ts b/web/src/features/public-api/server/index.tsnew file mode 100644index 0000000000..1e1b70fb86--- /dev/null+++ b/web/src/features/public-api/server/index.ts@@ -0,0 +1,10 @@+export { evaluationRulesApiHandler } from "./evaluation/evaluationRulesApiHandler";+export { evaluationRuleApiHandler } from "./evaluation/evaluationRuleApiHandler";+export { evaluatorsApiHandler } from "./evaluation/evaluatorsApiHandler";+export { evaluatorApiHandler } from "./evaluation/evaluatorApiHandler";+export { evaluatorVersionsApiHandler } from "./evaluation/evaluatorVersionsApiHandler";+export {+ createStructuredPublicApiRateLimitError,+ structuredPublicApiErrorContract,+ toStructuredPublicApiError,+} from "./structuredPublicApiErrorContract";diff --git a/web/src/features/public-api/server/stablePublicApiRoute.ts b/web/src/features/public-api/server/stablePublicApiRoute.tsnew file mode 100644index 0000000000..82b1771af7--- /dev/null+++ b/web/src/features/public-api/server/stablePublicApiRoute.ts@@ -0,0 +1,46 @@+import { type NextApiRequest, type NextApiResponse } from "next";+import { type ZodType } from "zod";+import {+ createAuthedProjectAPIRoute,+ type AuthedProjectAPIRouteConfig,+} from "@/src/features/public-api/server/createAuthedProjectAPIRoute";+import {+ withMiddlewares,+ type HttpMethod,+} from "@/src/features/public-api/server/withMiddlewares";+import { structuredPublicApiErrorContract } from "./structuredPublicApiErrorContract";++type StablePublicApiRouteConfig<+ TQuery extends ZodType<any>,+ TBody extends ZodType<any>,+ TResponse extends ZodType<any>,+> = Omit<+ AuthedProjectAPIRouteConfig<TQuery, TBody, TResponse>,+ "errorContract"+>;++type StablePublicApiHandlers = {+ [Method in HttpMethod]?: (+ req: NextApiRequest,+ res: NextApiResponse,+ ) => Promise<void>;+};++export const createStablePublicApiRoute = <+ TQuery extends ZodType<any>,+ TBody extends ZodType<any>,+ TResponse extends ZodType<any>,+>(+ routeConfig: StablePublicApiRouteConfig<TQuery, TBody, TResponse>,+) =>+ createAuthedProjectAPIRoute({+ ...routeConfig,+ errorContract: structuredPublicApiErrorContract,+ });++export const withStablePublicApiMiddlewares = (+ handlers: StablePublicApiHandlers,+) =>+ withMiddlewares(handlers, {+ errorContract: structuredPublicApiErrorContract,+ });diff --git a/web/src/features/public-api/server/unstable-public-api-error-contract.ts b/web/src/features/public-api/server/structuredPublicApiErrorContract.tssimilarity index 65%rename from web/src/features/public-api/server/unstable-public-api-error-contract.tsrename to web/src/features/public-api/server/structuredPublicApiErrorContract.tsindex 6454d5bb63..bd9e4b85d3 100644--- a/web/src/features/public-api/server/unstable-public-api-error-contract.ts+++ b/web/src/features/public-api/server/structuredPublicApiErrorContract.ts@@ -12,41 +12,31 @@ import {type RateLimitResult,} from "@langfuse/shared";import { ClickHouseResourceError } from "@langfuse/shared/src/server";+import {+ createStructuredPublicApiError,+ StructuredPublicApiError,+} from "@/src/features/public-api/types/structuredPublicApiError";import type {- UnstablePublicApiErrorCodeType,- UnstablePublicApiErrorDetailsType,-} from "@/src/features/public-api/shared/unstable-public-api-error-schema";+ StructuredPublicApiErrorCodeType,+ StructuredPublicApiErrorDetailsType,+} from "@/src/features/public-api/types/structuredPublicApiErrorSchema";import {getRateLimitUpgradeMessage,type RateLimitUpgradePath,} from "@/src/features/public-api/server/rateLimitUpgradePaths";-export const unstablePublicEvalsErrorContract = "unstable-public-evals";-export type PublicApiErrorContract = typeof unstablePublicEvalsErrorContract;+export const structuredPublicApiErrorContract = "structured";+export type PublicApiErrorContract = typeof structuredPublicApiErrorContract;-type UnstablePublicApiErrorBody = {+type StructuredPublicApiErrorBody = {message: string;- code: UnstablePublicApiErrorCodeType;- details?: UnstablePublicApiErrorDetailsType;+ code: StructuredPublicApiErrorCodeType;+ details?: StructuredPublicApiErrorDetailsType;};-export class UnstablePublicApiError extends BaseError {- public readonly code: UnstablePublicApiErrorCodeType;- public readonly details?: UnstablePublicApiErrorDetailsType;+export { StructuredPublicApiError };- constructor(params: {- httpCode: number;- code: UnstablePublicApiErrorCodeType;- message: string;- details?: UnstablePublicApiErrorDetailsType;- }) {- super("UnstablePublicApiError", params.httpCode, params.message, true);- this.code = params.code;- this.details = params.details;- }-}--function toBody(error: UnstablePublicApiError): UnstablePublicApiErrorBody {+function toBody(error: StructuredPublicApiError): StructuredPublicApiErrorBody {return {message: error.message,code: error.code,@@ -64,53 +54,44 @@ function toSerializableIssues(issues: ZodError["issues"]) {}));}-export function sendUnstablePublicApiErrorResponse(+export function sendStructuredPublicApiErrorResponse(res: NextApiResponse,- error: UnstablePublicApiError,+ error: StructuredPublicApiError,) {return res.status(error.httpCode).json(toBody(error));}-export function createUnstablePublicApiError(params: {- httpCode: number;- code: UnstablePublicApiErrorCodeType;- message: string;- details?: UnstablePublicApiErrorDetailsType;-}) {- return new UnstablePublicApiError(params);-}--export function createUnstablePublicApiAuthError(params: {+export function createStructuredPublicApiAuthError(params: {statusCode: number;message: string;}) {switch (params.statusCode) {case 400:- return createUnstablePublicApiError({+ return createStructuredPublicApiError({httpCode: 400,code: "invalid_request",message: params.message,});case 401:- return createUnstablePublicApiError({+ return createStructuredPublicApiError({httpCode: 401,code: "authentication_failed",message: params.message,});case 403:- return createUnstablePublicApiError({+ return createStructuredPublicApiError({httpCode: 403,code: "access_denied",message: params.message,});case 404:- return createUnstablePublicApiError({+ return createStructuredPublicApiError({httpCode: 404,code: "resource_not_found",message: params.message,});default:- return createUnstablePublicApiError({+ return createStructuredPublicApiError({httpCode: params.statusCode,code: params.statusCode >= 500 ? "internal_error" : "invalid_request",message: params.message,@@ -118,14 +99,14 @@ export function createUnstablePublicApiAuthError(params: {}}-export function createUnstablePublicApiRateLimitError(+export function createStructuredPublicApiRateLimitError(rateLimitRes: RateLimitResult,options?: {errorContract?: PublicApiErrorContract;upgradePath?: RateLimitUpgradePath;},) {- return createUnstablePublicApiError({+ return createStructuredPublicApiError({httpCode: 429,code: "rate_limited",message: options?.upgradePath@@ -140,11 +121,11 @@ export function createUnstablePublicApiRateLimitError(});}-export function createUnstablePublicApiRequestValidationError(params: {+export function createStructuredPublicApiRequestValidationError(params: {error: ZodError;requestPart: "query" | "body";}) {- return createUnstablePublicApiError({+ return createStructuredPublicApiError({httpCode: 400,code: params.requestPart === "query" ? "invalid_query" : "invalid_body",message:@@ -157,10 +138,10 @@ export function createUnstablePublicApiRequestValidationError(params: {});}-export function toUnstablePublicApiError(+export function toStructuredPublicApiError(error: unknown,-): UnstablePublicApiError {- if (error instanceof UnstablePublicApiError) {+): StructuredPublicApiError {+ if (error instanceof StructuredPublicApiError) {return error;}@@ -169,7 +150,7 @@ export function toUnstablePublicApiError(error.constructor.name === "ZodError" &&"issues" in error) {- return createUnstablePublicApiError({+ return createStructuredPublicApiError({httpCode: 400,code: "invalid_request",message: "Invalid request data",@@ -180,7 +161,7 @@ export function toUnstablePublicApiError(}if (error instanceof LangfuseNotFoundError) {- return createUnstablePublicApiError({+ return createStructuredPublicApiError({httpCode: 404,code: "resource_not_found",message: error.message,@@ -188,7 +169,7 @@ export function toUnstablePublicApiError(}if (error instanceof UnauthorizedError) {- return createUnstablePublicApiError({+ return createStructuredPublicApiError({httpCode: 403,code: "access_denied",message: error.message,@@ -196,7 +177,7 @@ export function toUnstablePublicApiError(}if (error instanceof ForbiddenError) {- return createUnstablePublicApiError({+ return createStructuredPublicApiError({httpCode: 403,code: "access_denied",message: error.message,@@ -204,7 +185,7 @@ export function toUnstablePublicApiError(}if (error instanceof MethodNotAllowedError) {- return createUnstablePublicApiError({+ return createStructuredPublicApiError({httpCode: 405,code: "method_not_allowed",message: error.message,@@ -212,7 +193,7 @@ export function toUnstablePublicApiError(}if (error instanceof LangfuseConflictError) {- return createUnstablePublicApiError({+ return createStructuredPublicApiError({httpCode: 409,code: "conflict",message: error.message,@@ -220,7 +201,7 @@ export function toUnstablePublicApiError(}if (error instanceof InvalidRequestError) {- return createUnstablePublicApiError({+ return createStructuredPublicApiError({httpCode: 400,code: "invalid_request",message: error.message,@@ -228,9 +209,9 @@ export function toUnstablePublicApiError(}if (error instanceof ClickHouseResourceError) {- return createUnstablePublicApiError({+ return createStructuredPublicApiError({httpCode: 422,- code: "unprocessable_content",+ code: "invalid_request",message: [ClickHouseResourceError.ERROR_ADVICE_MESSAGE,"See https://langfuse.com/docs/api-and-data-platform/features/public-api for more details.",@@ -239,7 +220,7 @@ export function toUnstablePublicApiError(}if (error instanceof InternalServerError) {- return createUnstablePublicApiError({+ return createStructuredPublicApiError({httpCode: 500,code: "internal_error",message: error.message,@@ -247,14 +228,14 @@ export function toUnstablePublicApiError(}if (error instanceof BaseError) {- return createUnstablePublicApiError({+ return createStructuredPublicApiError({httpCode: error.httpCode,code: error.httpCode >= 500 ? "internal_error" : "invalid_request",message: error.message,});}- return createUnstablePublicApiError({+ return createStructuredPublicApiError({httpCode: 500,code: "internal_error",message: "Internal Server Error",diff --git a/web/src/features/public-api/server/unstable-public-api-route.ts b/web/src/features/public-api/server/unstable-public-api-route.tsindex 659c7fd324..a341afc27f 100644--- a/web/src/features/public-api/server/unstable-public-api-route.ts+++ b/web/src/features/public-api/server/unstable-public-api-route.ts@@ -8,7 +8,7 @@ import {withMiddlewares,type HttpMethod,} from "@/src/features/public-api/server/withMiddlewares";-import { unstablePublicEvalsErrorContract } from "@/src/features/public-api/server/unstable-public-api-error-contract";+import { structuredPublicApiErrorContract } from "./structuredPublicApiErrorContract";type UnstablePublicApiRouteConfig<TQuery extends ZodType<any>,@@ -35,12 +35,12 @@ export const createUnstablePublicApiRoute = <) =>createAuthedProjectAPIRoute({...routeConfig,- errorContract: unstablePublicEvalsErrorContract,+ errorContract: structuredPublicApiErrorContract,});export const withUnstablePublicApiMiddlewares = (handlers: UnstablePublicApiHandlers,) =>withMiddlewares(handlers, {- errorContract: unstablePublicEvalsErrorContract,+ errorContract: structuredPublicApiErrorContract,});diff --git a/web/src/features/public-api/server/withMiddlewares.ts b/web/src/features/public-api/server/withMiddlewares.tsindex 9804e63af0..b7a90b0490 100644--- a/web/src/features/public-api/server/withMiddlewares.ts+++ b/web/src/features/public-api/server/withMiddlewares.ts@@ -17,11 +17,11 @@ import {} from "@langfuse/shared/src/server";import * as opentelemetry from "@opentelemetry/api";import {- sendUnstablePublicApiErrorResponse,- toUnstablePublicApiError,- unstablePublicEvalsErrorContract,+ sendStructuredPublicApiErrorResponse,+ structuredPublicApiErrorContract,+ toStructuredPublicApiError,type PublicApiErrorContract,-} from "@/src/features/public-api/server/unstable-public-api-error-contract";+} from "./structuredPublicApiErrorContract";import { clickHouseRouteForRequest } from "@/src/features/public-api/server/clickHouseRequestTags";// eslint-disable-next-line @typescript-eslint/no-unused-vars -- Used via typeof@@ -125,10 +125,10 @@ export function withMiddlewares(tags: error.tags,});- if (options?.errorContract === unstablePublicEvalsErrorContract) {- return sendUnstablePublicApiErrorResponse(+ if (options?.errorContract === structuredPublicApiErrorContract) {+ return sendStructuredPublicApiErrorResponse(res,- toUnstablePublicApiError(error),+ toStructuredPublicApiError(error),);}@@ -138,7 +138,7 @@ export function withMiddlewares(});}- if (options?.errorContract === unstablePublicEvalsErrorContract) {+ if (options?.errorContract === structuredPublicApiErrorContract) {if (isBaseError(error)) {logBaseError(error);} else if (isZodError(error)) {@@ -155,9 +155,9 @@ export function withMiddlewares(traceException(error);}- return sendUnstablePublicApiErrorResponse(+ return sendStructuredPublicApiErrorResponse(res,- toUnstablePublicApiError(error),+ toStructuredPublicApiError(error),);}diff --git a/web/src/features/public-api/types/evaluation/evaluationRules.ts b/web/src/features/public-api/types/evaluation/evaluationRules.tsnew file mode 100644index 0000000000..38a81b2612--- /dev/null+++ b/web/src/features/public-api/types/evaluation/evaluationRules.ts@@ -0,0 +1,111 @@+import { publicApiPaginationLimitZod } from "@langfuse/shared";+import { z } from "zod";+import {+ PromptVariableMapping,+ PromptVariableMappingInput,+ PublicEvaluationRuleReadFilter,+ StableEvaluationRuleFilter,+} from "./publicEvalsContract";+import {+ EncodedResourceCursor,+ EncodedResourceCursorString,+ PublicApiCreator,+} from "./evaluators";++const EvaluationRuleEvaluatorAssignmentInput = z+ .object({+ evaluatorId: z.string().min(1),+ variableMapping: z+ .array(PromptVariableMappingInput)+ .nullable()+ .default(null),+ })+ .strict();++const EvaluatorAssignment = z+ .object({+ evaluatorId: z.string(),+ variableMapping: z.array(PromptVariableMapping).nullable(),+ })+ .strict();++export const EvaluationRule = z+ .object({+ id: z.string(),+ name: z.string(),+ createdBy: PublicApiCreator.nullable(),+ enabled: z.boolean(),+ sampling: z.number().min(0).max(1),+ filter: z.array(PublicEvaluationRuleReadFilter),+ evaluatorAssignments: z.array(EvaluatorAssignment),+ createdAt: z.coerce.date(),+ updatedAt: z.coerce.date(),+ })+ .strict();++export const CreateEvaluationRuleBody = z+ .object({+ name: z.string().trim().min(1),+ enabled: z.boolean(),+ sampling: z.number().min(0).max(1).default(1),+ filter: z.array(StableEvaluationRuleFilter).default([]),+ evaluatorAssignments: z+ .array(EvaluationRuleEvaluatorAssignmentInput)+ .max(100),+ })+ .strict()+ .superRefine((value, ctx) => {+ if (value.enabled && value.evaluatorAssignments.length === 0) {+ ctx.addIssue({+ code: "custom",+ message:+ "An enabled evaluation rule requires at least one evaluator assignment",+ path: ["evaluatorAssignments"],+ });+ }+ });++export const UpdateEvaluationRuleBody = z+ .object({+ name: z.string().trim().min(1).optional(),+ enabled: z.boolean().optional(),+ sampling: z.number().min(0).max(1).optional(),+ filter: z.array(StableEvaluationRuleFilter).optional(),+ evaluatorAssignments: z+ .array(EvaluationRuleEvaluatorAssignmentInput)+ .max(100)+ .optional(),+ })+ .strict()+ .refine((value) => Object.keys(value).length > 0, {+ message: "Request body cannot be empty",+ });++export const ListEvaluationRulesQuery = z+ .object({+ limit: publicApiPaginationLimitZod,+ cursor: EncodedResourceCursor.optional(),+ })+ .strict();++export const ListEvaluationRulesResponse = z+ .object({+ data: z.array(EvaluationRule),+ meta: z.object({ cursor: EncodedResourceCursorString.optional() }).strict(),+ })+ .strict();++export const EvaluationRuleIdQuery = z+ .object({ evaluationRuleId: z.string() })+ .strict();++export const DeleteEvaluationRuleResponse = z+ .object({ id: z.string() })+ .strict();++export type CreateEvaluationRuleBodyType = z.infer<+ typeof CreateEvaluationRuleBody+>;+export type UpdateEvaluationRuleBodyType = z.infer<+ typeof UpdateEvaluationRuleBody+>;diff --git a/web/src/features/public-api/types/evaluation/evaluators.ts b/web/src/features/public-api/types/evaluation/evaluators.tsnew file mode 100644index 0000000000..0067ed1d46--- /dev/null+++ b/web/src/features/public-api/types/evaluation/evaluators.ts@@ -0,0 +1,241 @@+import {+ InvalidRequestError,+ publicApiPaginationLimitZod,+} from "@langfuse/shared";+import { z } from "zod";+import {+ PUBLIC_EVALUATOR_TYPE_CODE,+ PUBLIC_EVALUATOR_TYPE_LLM_AS_JUDGE,+ PromptVariableMapping,+ PublicCodeEvaluatorSourceCodeLanguage,+ PromptVariableMappingInput,+ PromptVariableMappingRead,+ PublicEvaluatorOutputDefinition,+ PublicEvaluatorOutputDefinitionRead,+} from "./publicEvalsContract";++export const PublicApiCreator = z+ .object({+ id: z.string(),+ name: z.string().nullable(),+ })+ .strict();++const EvaluatorChatMessage = z+ .object({+ role: z.literal("user"),+ content: z.string(),+ })+ .strict();++const EvaluatorChatPrompt = z.array(EvaluatorChatMessage).min(1).max(1);+const EvaluatorChatPromptInput = z.union([+ z+ .string()+ .min(1)+ .transform((content) => [{ role: "user" as const, content }]),+ EvaluatorChatPrompt,+]);++const EvaluatorVersionBase = z.object({+ id: z.string(),+ version: z.number().int().positive(),+ createdAt: z.coerce.date(),+ createdBy: PublicApiCreator.nullable(),+});++const EvaluatorModelConfig = z+ .object({+ provider: z.string().min(1),+ model: z.string().min(1),+ })+ .strict();++const LlmAsJudgeEvaluatorVersion = EvaluatorVersionBase.extend({+ type: z.literal(PUBLIC_EVALUATOR_TYPE_LLM_AS_JUDGE),+ prompt: EvaluatorChatPrompt,+ variables: z.array(z.string()),+ variableMapping: z.array(PromptVariableMappingRead).nullable(),+ modelConfig: EvaluatorModelConfig.nullable(),+ outputDefinition: PublicEvaluatorOutputDefinitionRead,+}).strict();++const CodeEvaluatorVersion = EvaluatorVersionBase.extend({+ type: z.literal(PUBLIC_EVALUATOR_TYPE_CODE),+ sourceCode: z.string().min(1),+ sourceCodeLanguage: PublicCodeEvaluatorSourceCodeLanguage,+}).strict();++export const EvaluatorVersion = z.discriminatedUnion("type", [+ LlmAsJudgeEvaluatorVersion,+ CodeEvaluatorVersion,+]);++const EvaluationRuleAssignment = z+ .object({+ evaluationRuleId: z.string(),+ variableMappingOverride: z.array(PromptVariableMapping).optional(),+ })+ .strict();++const EvaluatorBase = z.object({+ id: z.string(),+ name: z.string(),+ description: z.string().nullable(),+ createdBy: PublicApiCreator.nullable(),+ status: z.enum(["active", "paused"]),+ pausedAt: z.coerce.date().nullable(),+ pausedReason: z.string().nullable(),+ pausedMessage: z.string().nullable(),+ evaluationRuleAssignments: z.array(EvaluationRuleAssignment),+ createdAt: z.coerce.date(),+ updatedAt: z.coerce.date(),+ versionId: z.string(),+ version: z.number().int().positive(),+ versionCreatedAt: z.coerce.date(),+ versionCreatedBy: PublicApiCreator.nullable(),+});++export const LlmAsJudgeEvaluator = EvaluatorBase.extend({+ type: z.literal(PUBLIC_EVALUATOR_TYPE_LLM_AS_JUDGE),+ prompt: EvaluatorChatPrompt,+ variables: z.array(z.string()),+ variableMapping: z.array(PromptVariableMappingRead).nullable(),+ modelConfig: EvaluatorModelConfig.nullable(),+ outputDefinition: PublicEvaluatorOutputDefinitionRead,+}).strict();++const CodeEvaluator = EvaluatorBase.extend({+ type: z.literal(PUBLIC_EVALUATOR_TYPE_CODE),+ sourceCode: z.string().min(1),+ sourceCodeLanguage: PublicCodeEvaluatorSourceCodeLanguage,+}).strict();++export const Evaluator = z.discriminatedUnion("type", [+ LlmAsJudgeEvaluator,+ CodeEvaluator,+]);++const LlmAsJudgeEvaluatorDefinition = z+ .object({+ type: z.literal(PUBLIC_EVALUATOR_TYPE_LLM_AS_JUDGE),+ prompt: EvaluatorChatPromptInput,+ modelConfig: EvaluatorModelConfig.nullable().optional(),+ variableMapping: z.array(PromptVariableMappingInput).nullable().optional(),+ outputDefinition: PublicEvaluatorOutputDefinition,+ })+ .strict();++const CodeEvaluatorDefinition = z+ .object({+ type: z.literal(PUBLIC_EVALUATOR_TYPE_CODE),+ sourceCode: z.string().min(1),+ sourceCodeLanguage: PublicCodeEvaluatorSourceCodeLanguage,+ })+ .strict();++const _EvaluatorDefinition = z.discriminatedUnion("type", [+ LlmAsJudgeEvaluatorDefinition,+ CodeEvaluatorDefinition,+]);++const CreateEvaluatorMetadata = z.object({+ name: z.string().trim().min(1),+ description: z.string().trim().max(2_000).nullable().optional(),+});++export const CreateEvaluatorBody = z.discriminatedUnion("type", [+ CreateEvaluatorMetadata.extend(LlmAsJudgeEvaluatorDefinition.shape).strict(),+ CreateEvaluatorMetadata.extend(CodeEvaluatorDefinition.shape).strict(),+]);++const UpdateEvaluatorMetadata = z+ .object({+ name: z.string().trim().min(1).optional(),+ description: z.string().trim().max(2_000).nullable().optional(),+ })+ .strict();++export const UpdateEvaluatorBody = z.union([+ UpdateEvaluatorMetadata.extend(LlmAsJudgeEvaluatorDefinition.shape).strict(),+ UpdateEvaluatorMetadata.extend(CodeEvaluatorDefinition.shape).strict(),+ UpdateEvaluatorMetadata.refine((value) => Object.keys(value).length > 0, {+ message: "Request body cannot be empty",+ }),+]);++const decodeCursor = (value: string) => {+ try {+ return JSON.parse(Buffer.from(value, "base64url").toString("utf-8"));+ } catch (_error) {+ throw new InvalidRequestError("Invalid cursor format");+ }+};++const ResourceCursor = z.discriminatedUnion("v", [+ z.object({+ v: z.literal(1),+ lastCreatedAt: z.iso.datetime({ offset: true }),+ lastId: z.string(),+ }),+]);++export type ResourceCursorType = z.infer<typeof ResourceCursor>;++export const EncodedResourceCursor = z+ .string()+ .transform(decodeCursor)+ .pipe(ResourceCursor);++export const EncodedResourceCursorString = z+ .string()+ .describe("Opaque cursor for pagination");++export const encodeResourceCursor = (cursor: ResourceCursorType) =>+ Buffer.from(JSON.stringify(cursor)).toString("base64url");++const EvaluatorVersionCursor = z.discriminatedUnion("v", [+ z.object({ v: z.literal(1), version: z.number().int().positive() }),+]);++const EncodedEvaluatorVersionCursor = z+ .string()+ .transform(decodeCursor)+ .pipe(EvaluatorVersionCursor);++export const ListEvaluatorsQuery = z+ .object({+ limit: publicApiPaginationLimitZod,+ cursor: EncodedResourceCursor.optional(),+ })+ .strict();++export const ListEvaluatorsResponse = z+ .object({+ data: z.array(Evaluator),+ meta: z.object({ cursor: EncodedResourceCursorString.optional() }).strict(),+ })+ .strict();++export const EvaluatorIdQuery = z.object({ evaluatorId: z.string() }).strict();++export const ListEvaluatorVersionsQuery = z+ .object({+ evaluatorId: z.string(),+ limit: publicApiPaginationLimitZod,+ cursor: EncodedEvaluatorVersionCursor.optional(),+ })+ .strict();++export const ListEvaluatorVersionsResponse = z+ .object({+ data: z.array(EvaluatorVersion),+ meta: z.object({ cursor: EncodedResourceCursorString.optional() }).strict(),+ })+ .strict();++export const DeleteEvaluatorResponse = z.object({ id: z.string() }).strict();++export type EvaluatorDefinitionType = z.infer<typeof _EvaluatorDefinition>;+export type CreateEvaluatorBodyType = z.infer<typeof CreateEvaluatorBody>;+export type UpdateEvaluatorBodyType = z.infer<typeof UpdateEvaluatorBody>;diff --git a/web/src/features/public-api/types/evaluation/publicEvalsContract.ts b/web/src/features/public-api/types/evaluation/publicEvalsContract.tsnew file mode 100644index 0000000000..dd90d23a70--- /dev/null+++ b/web/src/features/public-api/types/evaluation/publicEvalsContract.ts@@ -0,0 +1,360 @@+import {+ arrayOptionsFilter,+ categoryOptionsFilter,+ nullFilter,+ numberFilter,+ numberObjectFilter,+ observationEvalFilterColumns,+ stringFilter,+ stringObjectFilter,+ stringOptionsFilter,+ timeFilter,+ booleanFilter,+ booleanObjectFilter,+ positionInTraceFilter,+ langfuseObjects,+} from "@langfuse/shared";+import { z } from "zod";++const PUBLIC_EVALUATOR_TYPES = ["llm_as_judge", "code"] as const;+export const [PUBLIC_EVALUATOR_TYPE_LLM_AS_JUDGE, PUBLIC_EVALUATOR_TYPE_CODE] =+ PUBLIC_EVALUATOR_TYPES;++const PublicEvaluatorType = z.enum(PUBLIC_EVALUATOR_TYPES);+export const PublicCodeEvaluatorSourceCodeLanguage = z.enum([+ "PYTHON",+ "TYPESCRIPT",+]);++const _PublicEvaluatorModelConfig = z.object({+ provider: z.string().min(1),+ model: z.string().min(1),+});++const PublicEvaluatorOutputDescription = z.string().trim().min(1).optional();++const PublicEvaluatorOutputDefinitionBase = z.object({+ scoreReasoningInstructions: PublicEvaluatorOutputDescription,+ scoreValueInstructions: PublicEvaluatorOutputDescription,+});++const PublicEvaluatorNumericScore = PublicEvaluatorOutputDefinitionBase.extend({+ dataType: z.literal("NUMERIC"),+ minValue: z.number().optional(),+ maxValue: z.number().optional(),+}).refine(+ ({ minValue, maxValue }) =>+ minValue === undefined || maxValue === undefined || minValue <= maxValue,+ {+ message: "Minimum value must be less than or equal to maximum value",+ },+);++const PublicEvaluatorBooleanScore = PublicEvaluatorOutputDefinitionBase.extend({+ dataType: z.literal("BOOLEAN"),+});++const PublicEvaluatorCategoricalScore =+ PublicEvaluatorOutputDefinitionBase.extend({+ dataType: z.literal("CATEGORICAL"),+ categories: z.array(z.string().trim().min(1)).min(2),+ shouldAllowMultipleMatches: z.boolean(),+ });++export const PublicEvaluatorOutputDefinition = z.discriminatedUnion(+ "dataType",+ [+ PublicEvaluatorNumericScore,+ PublicEvaluatorBooleanScore,+ PublicEvaluatorCategoricalScore,+ ],+);++const PublicEvaluatorOutputDefinitionReadBase = z.object({+ scoreReasoningInstructions: z.string().optional(),+ scoreValueInstructions: z.string().optional(),+});++export const PublicEvaluatorOutputDefinitionRead = z.discriminatedUnion(+ "dataType",+ [+ PublicEvaluatorOutputDefinitionReadBase.extend({+ dataType: z.literal("NUMERIC"),+ minValue: z.number().optional(),+ maxValue: z.number().optional(),+ }),+ PublicEvaluatorOutputDefinitionReadBase.extend({+ dataType: z.literal("BOOLEAN"),+ }),+ PublicEvaluatorOutputDefinitionReadBase.extend({+ dataType: z.literal("CATEGORICAL"),+ categories: z.array(z.string()),+ shouldAllowMultipleMatches: z.boolean(),+ }),+ ],+);++const PublicEvaluationRuleTarget = z.enum(["observation", "experiment"]);+const PublicEvaluationRuleLegacyTarget = z.enum(["trace", "dataset"]);+const _PublicEvaluationRuleReadTarget = z.union([+ PublicEvaluationRuleTarget,+ PublicEvaluationRuleLegacyTarget,+]);++const _PublicEvaluationRuleStatus = z.enum(["active", "inactive", "paused"]);++// No `scope`: every evaluator is project-owned, so the field could only ever+// carry one value. Gallery templates are starter definitions that are copied+// into the project on save, not addressable Langfuse-managed resources.+const PublicEvaluationRuleEvaluatorReference = z.object({+ name: z.string().min(1),+ type: PublicEvaluatorType.default(PUBLIC_EVALUATOR_TYPE_LLM_AS_JUDGE),+});++// PATCH intentionally omits `type`: an evaluation rule's evaluator type cannot+// be changed. The service always inherits the rule's current type, so a code+// rule is never retargeted to an LLM evaluator family, which stores its+// mapping per assignment and would inherit none. To use a different evaluator+// type, create a new rule.+const _PublicEvaluationRuleEvaluator =+ PublicEvaluationRuleEvaluatorReference.extend({+ id: z.string(),+ });++const ObservationPromptVariableMappingSource = z.enum([+ "input",+ "output",+ "metadata",+ "tool_calls",+]);++const ExperimentPromptVariableMappingSource = z.enum([+ "input",+ "output",+ "metadata",+ "tool_calls",+ "expected_output",+ "experiment_item_metadata",+]);++function createMappingSchema<+ TSource extends z.ZodType<+ | "input"+ | "output"+ | "metadata"+ | "tool_calls"+ | "expected_output"+ | "experiment_item_metadata"+ >,+>(sourceSchema: TSource) {+ return z.object({+ variable: z.string().min(1),+ source: sourceSchema,+ jsonPath: z.string().min(1).optional(),+ });+}++const ObservationPromptVariableMappingInput = createMappingSchema(+ ObservationPromptVariableMappingSource,+);++const ExperimentPromptVariableMappingInput = createMappingSchema(+ ExperimentPromptVariableMappingSource,+);++export const PromptVariableMappingInput = z.union([+ ObservationPromptVariableMappingInput,+ ExperimentPromptVariableMappingInput,+]);++// This shape preserves incomplete mappings so callers can repair them.+// Mapping inputs remain strict and require a concrete source.+export const PromptVariableMappingRead = z.object({+ variable: z.string().min(1),+ source: ExperimentPromptVariableMappingSource.nullable(),+ jsonPath: z.string().min(1).optional(),+});++export const LegacyPromptVariableMapping = z+ .object({+ mappingType: z.literal("legacy"),+ variable: z.string().min(1),+ langfuseObject: z.enum(langfuseObjects),+ objectName: z.string().nullable(),+ source: z.string().min(1),+ jsonPath: z.string().min(1).optional(),+ })+ .refine(+ (mapping) =>+ mapping.langfuseObject === "trace" ||+ mapping.langfuseObject === "dataset_item" ||+ mapping.objectName !== null,+ {+ path: ["objectName"],+ message: "objectName is required for observation objects",+ },+ );++export const PromptVariableMapping = z.union([+ LegacyPromptVariableMapping,+ PromptVariableMappingRead,+]);++const PublicEvaluationRuleReadFilterBase = z+ .object({+ type: z.string(),+ column: z.string(),+ operator: z.string(),+ value: z.unknown().optional(),+ })+ .loose();++export const PublicEvaluationRuleReadFilter = z.union([+ PublicEvaluationRuleReadFilterBase.safeExtend({ key: z.string() }),+ PublicEvaluationRuleReadFilterBase,+]);++const filterSchemaFactories = {+ datetime: (columnId: string) =>+ timeFilter.safeExtend({ column: z.literal(columnId) }),+ string: (columnId: string) =>+ stringFilter.safeExtend({ column: z.literal(columnId) }),+ number: (columnId: string) =>+ numberFilter.safeExtend({ column: z.literal(columnId) }),+ stringOptions: (columnId: string) =>+ stringOptionsFilter.safeExtend({ column: z.literal(columnId) }),+ categoryOptions: (columnId: string) =>+ categoryOptionsFilter.safeExtend({ column: z.literal(columnId) }),+ arrayOptions: (columnId: string) =>+ arrayOptionsFilter.safeExtend({ column: z.literal(columnId) }),+ stringObject: (columnId: string) =>+ stringObjectFilter.safeExtend({ column: z.literal(columnId) }),+ numberObject: (columnId: string) =>+ numberObjectFilter.safeExtend({ column: z.literal(columnId) }),+ boolean: (columnId: string) =>+ booleanFilter.safeExtend({ column: z.literal(columnId) }),+ booleanObject: (columnId: string) =>+ booleanObjectFilter.safeExtend({ column: z.literal(columnId) }),+ null: (columnId: string) =>+ nullFilter.safeExtend({ column: z.literal(columnId) }),+ positionInTrace: (columnId: string) =>+ positionInTraceFilter.safeExtend({ column: z.literal(columnId) }),+} as const;++type SupportedFilterFactory = keyof typeof filterSchemaFactories;++function createTargetFilterSchema(+ columns: Array<{ id: string; type: SupportedFilterFactory }>,+) {+ const schemas = columns.map((column) =>+ filterSchemaFactories[column.type](column.id),+ );++ if (schemas.length === 1) {+ return schemas[0]!;+ }++ return z.union(+ schemas as [+ (typeof schemas)[number],+ (typeof schemas)[number],+ ...Array<(typeof schemas)[number]>,+ ],+ );+}++// The experiment-root filter is what the `experiment` target *means*, so it is+// not separately addressable: exposing it on `observation` too would give one+// rule two contradictory public representations.+const EXPERIMENT_ROOT_FILTER_COLUMN = "isExperimentItemRootSpan";++const OBSERVATION_EVALUATION_RULE_FILTER_COLUMNS = observationEvalFilterColumns+ .filter((column) => column.id !== EXPERIMENT_ROOT_FILTER_COLUMN)+ .map((column) => ({+ id: column.id,+ type: column.type,+ }));++// An experiment rule is an observation rule scoped to experiment root spans,+// so it accepts every observation filter plus the dataset scope.+const EXPERIMENT_EVALUATION_RULE_FILTER_COLUMNS = [+ ...OBSERVATION_EVALUATION_RULE_FILTER_COLUMNS,+ {+ id: "datasetId",+ type: "stringOptions" as const,+ },+];++const ObservationEvaluationRuleFilter = createTargetFilterSchema(+ OBSERVATION_EVALUATION_RULE_FILTER_COLUMNS,+);++const ExperimentEvaluationRuleFilter = createTargetFilterSchema(+ EXPERIMENT_EVALUATION_RULE_FILTER_COLUMNS,+);++const _PublicEvaluationRuleFilter = z.union([+ ObservationEvaluationRuleFilter,+ ExperimentEvaluationRuleFilter,+]);++// Stable evaluation rules are all event rules. Experiment scope is expressed+// by the isExperimentItemRootSpan filter rather than a separate target field.+const STABLE_EVALUATION_RULE_FILTER_COLUMNS = [+ ...observationEvalFilterColumns.map((column) => ({+ id: column.id,+ type: column.type,+ })),+ {+ id: "datasetId",+ type: "stringOptions" as const,+ },+];++export const StableEvaluationRuleFilter = createTargetFilterSchema(+ STABLE_EVALUATION_RULE_FILTER_COLUMNS,+);++export type PublicEvaluatorModelConfigType = z.infer<+ typeof _PublicEvaluatorModelConfig+>;+export type PublicEvaluatorOutputDefinitionType = z.infer<+ typeof PublicEvaluatorOutputDefinition+>;+export type PublicEvaluatorTypeType = z.infer<typeof PublicEvaluatorType>;+export type PublicEvaluationRuleTargetType = z.infer<+ typeof PublicEvaluationRuleTarget+>;+export type PublicEvaluationRuleLegacyTargetType = z.infer<+ typeof PublicEvaluationRuleLegacyTarget+>;+export type PublicEvaluationRuleReadTargetType = z.infer<+ typeof _PublicEvaluationRuleReadTarget+>;+export type PublicEvaluationRuleStatusType = z.infer<+ typeof _PublicEvaluationRuleStatus+>;+export type PublicEvaluationRuleEvaluatorReferenceType = z.infer<+ typeof PublicEvaluationRuleEvaluatorReference+>;+export type PublicEvaluationRuleEvaluatorType = z.infer<+ typeof _PublicEvaluationRuleEvaluator+>;+export type PromptVariableMappingInputType = z.infer<+ typeof PromptVariableMappingInput+>;+export type PromptVariableMappingReadType = z.infer<+ typeof PromptVariableMappingRead+>;+export type ObservationPromptVariableMappingInputType = z.infer<+ typeof ObservationPromptVariableMappingInput+>;+export type LegacyPromptVariableMappingType = z.infer<+ typeof LegacyPromptVariableMapping+>;+export type PublicEvaluationRuleFilterType = z.infer<+ typeof _PublicEvaluationRuleFilter+>;+export type PublicCodeEvaluatorSourceCodeLanguageType = z.infer<+ typeof PublicCodeEvaluatorSourceCodeLanguage+>;diff --git a/web/src/features/public-api/types/public-api-errors.ts b/web/src/features/public-api/types/public-api-errors.tsnew file mode 100644index 0000000000..a246c89c3a--- /dev/null+++ b/web/src/features/public-api/types/public-api-errors.ts@@ -0,0 +1,25 @@+import { z } from "zod";+import { StructuredPublicApiErrorDetails } from "./structuredPublicApiErrorSchema";++const publicApiErrorCodes = [+ "authentication_failed",+ "access_denied",+ "invalid_request",+ "invalid_query",+ "invalid_body",+ "resource_not_found",+ "conflict",+ "rate_limited",+ "method_not_allowed",+ "internal_error",+] as const;++const PublicApiErrorCode = z.enum(publicApiErrorCodes);++export const PublicApiError = z+ .object({+ message: z.string(),+ code: PublicApiErrorCode,+ details: StructuredPublicApiErrorDetails.optional(),+ })+ .strict();diff --git a/web/src/features/public-api/types/structuredPublicApiError.ts b/web/src/features/public-api/types/structuredPublicApiError.tsnew file mode 100644index 0000000000..1ba5a7b01a--- /dev/null+++ b/web/src/features/public-api/types/structuredPublicApiError.ts@@ -0,0 +1,30 @@+import { BaseError } from "@langfuse/shared";+import type {+ StructuredPublicApiErrorCodeType,+ StructuredPublicApiErrorDetailsType,+} from "./structuredPublicApiErrorSchema";++export class StructuredPublicApiError extends BaseError {+ public readonly code: StructuredPublicApiErrorCodeType;+ public readonly details?: StructuredPublicApiErrorDetailsType;++ constructor(params: {+ httpCode: number;+ code: StructuredPublicApiErrorCodeType;+ message: string;+ details?: StructuredPublicApiErrorDetailsType;+ }) {+ super("StructuredPublicApiError", params.httpCode, params.message, true);+ this.code = params.code;+ this.details = params.details;+ }+}++export function createStructuredPublicApiError(params: {+ httpCode: number;+ code: StructuredPublicApiErrorCodeType;+ message: string;+ details?: StructuredPublicApiErrorDetailsType;+}) {+ return new StructuredPublicApiError(params);+}diff --git a/web/src/features/public-api/shared/unstable-public-api-error-schema.ts b/web/src/features/public-api/types/structuredPublicApiErrorSchema.tssimilarity index 68%rename from web/src/features/public-api/shared/unstable-public-api-error-schema.tsrename to web/src/features/public-api/types/structuredPublicApiErrorSchema.tsindex 056aab0cf8..49d39de7f6 100644--- a/web/src/features/public-api/shared/unstable-public-api-error-schema.ts+++ b/web/src/features/public-api/types/structuredPublicApiErrorSchema.ts@@ -1,6 +1,6 @@import { z } from "zod";-const unstablePublicApiErrorCodes = [+const structuredPublicApiErrorCodes = ["authentication_failed","access_denied","invalid_request",@@ -22,9 +22,9 @@ const unstablePublicApiErrorCodes = ["internal_error",] as const;-const UnstablePublicApiErrorCode = z.enum(unstablePublicApiErrorCodes);+const StructuredPublicApiErrorCode = z.enum(structuredPublicApiErrorCodes);-const UnstablePublicApiValidationIssue = z+const StructuredPublicApiValidationIssue = z.object({code: z.string(),message: z.string(),@@ -32,9 +32,9 @@ const UnstablePublicApiValidationIssue = z}).loose();-const UnstablePublicApiErrorDetails = z+export const StructuredPublicApiErrorDetails = z.object({- issues: z.array(UnstablePublicApiValidationIssue).optional(),+ issues: z.array(StructuredPublicApiValidationIssue).optional(),field: z.string().optional(),column: z.string().optional(),invalidValues: z.array(z.string()).optional(),@@ -52,17 +52,17 @@ const UnstablePublicApiErrorDetails = z}).strict();-export const UnstablePublicApiErrorResponse = z+export const StructuredPublicApiErrorResponse = z.object({message: z.string(),- code: UnstablePublicApiErrorCode,- details: UnstablePublicApiErrorDetails.optional(),+ code: StructuredPublicApiErrorCode,+ details: StructuredPublicApiErrorDetails.optional(),}).strict();-export type UnstablePublicApiErrorCodeType = z.infer<- typeof UnstablePublicApiErrorCode+export type StructuredPublicApiErrorCodeType = z.infer<+ typeof StructuredPublicApiErrorCode>;-export type UnstablePublicApiErrorDetailsType = z.infer<- typeof UnstablePublicApiErrorDetails+export type StructuredPublicApiErrorDetailsType = z.infer<+ typeof StructuredPublicApiErrorDetails>;diff --git a/web/src/features/public-api/types/unstable-evaluation-rules.ts b/web/src/features/public-api/types/unstable-evaluation-rules.tsindex 22bf83b6a8..69fa6d6fb7 100644--- a/web/src/features/public-api/types/unstable-evaluation-rules.ts+++ b/web/src/features/public-api/types/unstable-evaluation-rules.ts@@ -2,16 +2,16 @@ import { JobTimeScopeZod, singleFilter } from "@langfuse/shared";import { z } from "zod";import {ExperimentEvaluationRuleFilter,- ExperimentEvaluationRuleMapping,- LegacyEvaluationRuleMapping,+ ExperimentPromptVariableMappingInput,+ LegacyPromptVariableMapping,ObservationEvaluationRuleFilter,- ObservationEvaluationRuleMapping,+ ObservationPromptVariableMappingInput,PUBLIC_EVALUATOR_TYPE_CODE,PublicEvaluationRuleFilter,PublicEvaluationRuleEvaluator,PublicEvaluationRuleEvaluatorReference,PublicEvaluationRuleEvaluatorReferencePatch,- PublicEvaluationRuleReadMapping,+ PromptVariableMappingRead,PublicEvaluationRuleLegacyTarget,PublicEvaluationRuleStatus,PublicEvaluationRuleTarget,@@ -39,12 +39,12 @@ const APIEvaluationRule = zevaluators: z.array(z.object({evaluator: PublicEvaluationRuleEvaluator,- mapping: z.array(PublicEvaluationRuleReadMapping).nullable(),+ mapping: z.array(PromptVariableMappingRead).nullable(),}),),target: PublicEvaluationRuleTarget,filter: z.array(PublicEvaluationRuleFilter),- mapping: z.array(PublicEvaluationRuleReadMapping),+ mapping: z.array(PromptVariableMappingRead),}).strict();@@ -54,14 +54,14 @@ const APILegacyEvaluationRule = zevaluators: z.array(z.object({evaluator: PublicEvaluationRuleEvaluator,- mapping: z.array(LegacyEvaluationRuleMapping).nullable(),+ mapping: z.array(LegacyPromptVariableMapping).nullable(),}),),target: PublicEvaluationRuleLegacyTarget,delay: z.number().int().nonnegative(),timeScope: z.array(JobTimeScopeZod),filter: z.array(singleFilter),- mapping: z.array(LegacyEvaluationRuleMapping),+ mapping: z.array(LegacyPromptVariableMapping),}).strict();@@ -103,25 +103,25 @@ function evaluatorAssignment<T extends z.ZodType>(mapping: T) {const PostUnstableObservationEvaluationRuleBody = z.object({...EvaluationRuleCreateBase,evaluators: z- .array(evaluatorAssignment(ObservationEvaluationRuleMapping))+ .array(evaluatorAssignment(ObservationPromptVariableMappingInput)).min(1).max(100).optional(),target: z.literal("observation"),filter: z.array(ObservationEvaluationRuleFilter).default([]),- mapping: z.array(ObservationEvaluationRuleMapping).optional(),+ mapping: z.array(ObservationPromptVariableMappingInput).optional(),});const PostUnstableExperimentEvaluationRuleBody = z.object({...EvaluationRuleCreateBase,evaluators: z- .array(evaluatorAssignment(ExperimentEvaluationRuleMapping))+ .array(evaluatorAssignment(ExperimentPromptVariableMappingInput)).min(1).max(100).optional(),target: z.literal("experiment"),filter: z.array(ExperimentEvaluationRuleFilter).default([]),- mapping: z.array(ExperimentEvaluationRuleMapping).optional(),+ mapping: z.array(ExperimentPromptVariableMappingInput).optional(),});// `code` evaluators use a fixed runtime mapping managed by Langfuse. LLM@@ -196,7 +196,7 @@ const EXCLUSIVE_EVALUATOR_FIELDS_MESSAGE =const UntargetedEvaluationRulePatch = z.object({...EvaluationRulePatchBase,- evaluators: patchEvaluators(ObservationEvaluationRuleMapping),+ evaluators: patchEvaluators(ObservationPromptVariableMappingInput),target: z.undefined().optional(),filter: z.undefined().optional(),mapping: z.undefined().optional(),@@ -204,18 +204,18 @@ const UntargetedEvaluationRulePatch = z.object({const ObservationEvaluationRulePatch = z.object({...EvaluationRulePatchBase,- evaluators: patchEvaluators(ObservationEvaluationRuleMapping),+ evaluators: patchEvaluators(ObservationPromptVariableMappingInput),target: z.literal("observation"),filter: z.array(ObservationEvaluationRuleFilter).optional(),- mapping: z.array(ObservationEvaluationRuleMapping).optional(),+ mapping: z.array(ObservationPromptVariableMappingInput).optional(),});const ExperimentEvaluationRulePatch = z.object({...EvaluationRulePatchBase,- evaluators: patchEvaluators(ExperimentEvaluationRuleMapping),+ evaluators: patchEvaluators(ExperimentPromptVariableMappingInput),target: z.literal("experiment"),filter: z.array(ExperimentEvaluationRuleFilter).optional(),- mapping: z.array(ExperimentEvaluationRuleMapping).optional(),+ mapping: z.array(ExperimentPromptVariableMappingInput).optional(),});export const PatchUnstableEvaluationRuleBody = zdiff --git a/web/src/features/public-api/types/unstable-evaluators.ts b/web/src/features/public-api/types/unstable-evaluators.tsindex 3ea2076a36..74779a0788 100644--- a/web/src/features/public-api/types/unstable-evaluators.ts+++ b/web/src/features/public-api/types/unstable-evaluators.ts@@ -2,8 +2,8 @@ import { z } from "zod";import {PUBLIC_EVALUATOR_TYPE_CODE,PUBLIC_EVALUATOR_TYPE_LLM_AS_JUDGE,- ObservationEvaluationRuleMapping,- PublicEvaluationRuleReadMapping,+ ObservationPromptVariableMappingInput,+ PromptVariableMappingRead,PublicCodeEvaluatorDefinitionInput,PublicEvaluatorModelConfig,PublicEvaluatorOutputDefinition,@@ -20,7 +20,7 @@ const APIEvaluatorBase = zvariables: z.array(z.string()),// An evaluator's default mapping can name experiment-only sources, and a legacy one can be// incomplete, so reads use the permissive schema. Requests stay strict.- mapping: z.array(PublicEvaluationRuleReadMapping).nullable(),+ mapping: z.array(PromptVariableMappingRead).nullable(),evaluationRuleCount: z.number().int().nonnegative(),createdAt: z.coerce.date(),updatedAt: z.coerce.date(),@@ -81,7 +81,7 @@ const PostUnstableLlmAsJudgeEvaluatorBody = z.object({...EvaluatorCreateBase,type: z.literal(PUBLIC_EVALUATOR_TYPE_LLM_AS_JUDGE),...PublicLlmAsJudgeEvaluatorDefinitionInput.shape,- mapping: z.array(ObservationEvaluationRuleMapping).optional(),+ mapping: z.array(ObservationPromptVariableMappingInput).optional(),sourceCode: z.never().optional(),sourceCodeLanguage: z.never().optional(),});diff --git a/web/src/features/public-api/types/unstable-public-evals-contract.ts b/web/src/features/public-api/types/unstable-public-evals-contract.tsindex ce26632e84..09fdab098b 100644--- a/web/src/features/public-api/types/unstable-public-evals-contract.ts+++ b/web/src/features/public-api/types/unstable-public-evals-contract.ts@@ -12,15 +12,15 @@ import {timeFilter,experimentEvalFilterColumns,booleanFilter,+ CODE_EVAL_SOURCE_MAX_BYTES,langfuseObjects,} from "@langfuse/shared";-import { CODE_EVAL_SOURCE_MAX_BYTES } from "@langfuse/shared/src/server";import { z } from "zod";-export { UnstablePublicApiErrorResponse } from "@/src/features/public-api/shared/unstable-public-api-error-schema";+export { StructuredPublicApiErrorResponse } from "./structuredPublicApiErrorSchema";import type {- UnstablePublicApiErrorCodeType,- UnstablePublicApiErrorDetailsType,-} from "@/src/features/public-api/shared/unstable-public-api-error-schema";+ StructuredPublicApiErrorCodeType,+ StructuredPublicApiErrorDetailsType,+} from "./structuredPublicApiErrorSchema";const PUBLIC_EVALUATOR_TYPES = ["llm_as_judge", "code"] as const;export const [PUBLIC_EVALUATOR_TYPE_LLM_AS_JUDGE, PUBLIC_EVALUATOR_TYPE_CODE] =@@ -85,9 +85,7 @@ export const PublicEvaluatorOutputDefinition = z.discriminatedUnion(export const PublicEvaluationRuleTarget = z.enum(["observation", "experiment"]);export const PublicEvaluationRuleLegacyTarget = z.enum(["trace", "dataset"]);--// eslint-disable-next-line @typescript-eslint/no-unused-vars -- Used via z.infer-const PublicEvaluationRuleReadTarget = z.union([+const _PublicEvaluationRuleReadTarget = z.union([PublicEvaluationRuleTarget,PublicEvaluationRuleLegacyTarget,]);@@ -120,14 +118,14 @@ export const PublicEvaluationRuleEvaluator =id: z.string(),});-export const ObservationEvaluationRuleMappingSource = z.enum([+export const ObservationPromptVariableMappingSource = z.enum(["input","output","metadata","tool_calls",]);-export const ExperimentEvaluationRuleMappingSource = z.enum([+export const ExperimentPromptVariableMappingSource = z.enum(["input","output","metadata",@@ -153,29 +151,28 @@ function createMappingSchema<});}-export const ObservationEvaluationRuleMapping = createMappingSchema(- ObservationEvaluationRuleMappingSource,+export const ObservationPromptVariableMappingInput = createMappingSchema(+ ObservationPromptVariableMappingSource,);-export const ExperimentEvaluationRuleMapping = createMappingSchema(- ExperimentEvaluationRuleMappingSource,+export const ExperimentPromptVariableMappingInput = createMappingSchema(+ ExperimentPromptVariableMappingSource,);-// eslint-disable-next-line @typescript-eslint/no-unused-vars -- Used via z.infer-const PublicEvaluationRuleMapping = z.union([- ObservationEvaluationRuleMapping,- ExperimentEvaluationRuleMapping,+const _PromptVariableMappingInput = z.union([+ ObservationPromptVariableMappingInput,+ ExperimentPromptVariableMappingInput,]);-// Read responses preserve incomplete legacy mappings so callers can repair-// them. Write schemas remain strict and require a concrete source.-export const PublicEvaluationRuleReadMapping = z.object({+// This shape preserves incomplete mappings so callers can repair them.+// Mapping inputs remain strict and require a concrete source.+export const PromptVariableMappingRead = z.object({variable: z.string().min(1),- source: ExperimentEvaluationRuleMappingSource.nullable(),+ source: ExperimentPromptVariableMappingSource.nullable(),jsonPath: z.string().min(1).optional(),});-export const LegacyEvaluationRuleMapping = z+export const LegacyPromptVariableMapping = z.object({variable: z.string().min(1),langfuseObject: z.enum(langfuseObjects),@@ -289,7 +286,7 @@ export type PublicEvaluationRuleLegacyTargetType = z.infer<typeof PublicEvaluationRuleLegacyTarget>;export type PublicEvaluationRuleReadTargetType = z.infer<- typeof PublicEvaluationRuleReadTarget+ typeof _PublicEvaluationRuleReadTarget>;export type PublicEvaluationRuleStatusType = z.infer<typeof PublicEvaluationRuleStatus@@ -300,24 +297,24 @@ export type PublicEvaluationRuleEvaluatorReferenceType = z.infer<export type PublicEvaluationRuleEvaluatorType = z.infer<typeof PublicEvaluationRuleEvaluator>;-export type PublicEvaluationRuleMappingType = z.infer<- typeof PublicEvaluationRuleMapping+export type PromptVariableMappingInputType = z.infer<+ typeof _PromptVariableMappingInput>;-export type PublicEvaluationRuleReadMappingType = z.infer<- typeof PublicEvaluationRuleReadMapping+export type PromptVariableMappingReadType = z.infer<+ typeof PromptVariableMappingRead>;-export type PublicObservationEvaluationRuleMappingType = z.infer<- typeof ObservationEvaluationRuleMapping+export type ObservationPromptVariableMappingInputType = z.infer<+ typeof ObservationPromptVariableMappingInput>;-export type LegacyEvaluationRuleMappingType = z.infer<- typeof LegacyEvaluationRuleMapping+export type LegacyPromptVariableMappingType = z.infer<+ typeof LegacyPromptVariableMapping>;export type PublicEvaluationRuleFilterType = z.infer<typeof PublicEvaluationRuleFilter>;export type {- UnstablePublicApiErrorCodeType,- UnstablePublicApiErrorDetailsType,+ StructuredPublicApiErrorCodeType,+ StructuredPublicApiErrorDetailsType,};export const UnstablePublicApiPaginationQuery = z.object({diff --git a/web/src/features/widgets/server/public-dashboard-widget-service.ts b/web/src/features/widgets/server/public-dashboard-widget-service.tsindex 7ea9587baf..d6a6fea15e 100644--- a/web/src/features/widgets/server/public-dashboard-widget-service.ts+++ b/web/src/features/widgets/server/public-dashboard-widget-service.ts@@ -14,7 +14,7 @@ import {type ViewVersion,} from "@langfuse/shared/query";import { auditLog } from "@/src/features/audit-logs/auditLog";-import { createUnstablePublicApiError } from "@/src/features/public-api/server/unstable-public-api-error-contract";+import { createStructuredPublicApiError } from "@/src/features/public-api";import {PostUnstableDashboardWidgetResponse,type DashboardWidgetViewOutputType,@@ -42,7 +42,7 @@ const throwInvalidWidget = (params: {field?: string;allowedValues?: string[];}): never => {- throw createUnstablePublicApiError({+ throw createStructuredPublicApiError({httpCode: 400,code: "invalid_request",message: params.message,diff --git a/web/src/pages/api/public/v2/evaluation-rules/[evaluationRuleId].ts b/web/src/pages/api/public/v2/evaluation-rules/[evaluationRuleId].tsnew file mode 100644index 0000000000..64c1ac60f9--- /dev/null+++ b/web/src/pages/api/public/v2/evaluation-rules/[evaluationRuleId].ts@@ -0,0 +1 @@+export { evaluationRuleApiHandler as default } from "@/src/features/public-api/server";diff --git a/web/src/pages/api/public/v2/evaluation-rules/index.ts b/web/src/pages/api/public/v2/evaluation-rules/index.tsnew file mode 100644index 0000000000..336cda3b18--- /dev/null+++ b/web/src/pages/api/public/v2/evaluation-rules/index.ts@@ -0,0 +1 @@+export { evaluationRulesApiHandler as default } from "@/src/features/public-api/server";diff --git a/web/src/pages/api/public/v2/evaluators/[evaluatorId].ts b/web/src/pages/api/public/v2/evaluators/[evaluatorId].tsnew file mode 100644index 0000000000..6f7f385879--- /dev/null+++ b/web/src/pages/api/public/v2/evaluators/[evaluatorId].ts@@ -0,0 +1 @@+export { evaluatorApiHandler as default } from "@/src/features/public-api/server";diff --git a/web/src/pages/api/public/v2/evaluators/[evaluatorId]/versions.ts b/web/src/pages/api/public/v2/evaluators/[evaluatorId]/versions.tsnew file mode 100644index 0000000000..ce6ce09921--- /dev/null+++ b/web/src/pages/api/public/v2/evaluators/[evaluatorId]/versions.ts@@ -0,0 +1 @@+export { evaluatorVersionsApiHandler as default } from "@/src/features/public-api/server";diff --git a/web/src/pages/api/public/v2/evaluators/index.ts b/web/src/pages/api/public/v2/evaluators/index.tsnew file mode 100644index 0000000000..632118bf32--- /dev/null+++ b/web/src/pages/api/public/v2/evaluators/index.ts@@ -0,0 +1 @@+export { evaluatorsApiHandler as default } from "@/src/features/public-api/server";