Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(aio): add a trend insight for sentiment evaluations (#126196)

Bernat Torres4cb88ec63ec3
diff --git a/products/ai_observability/frontend/evaluations/constants.ts b/products/ai_observability/frontend/evaluations/constants.ts
index e271b0f620bb..48f9896f496d 100644
--- a/products/ai_observability/frontend/evaluations/constants.ts
+++ b/products/ai_observability/frontend/evaluations/constants.ts
@@ -74,13 +74,17 @@ export function numericOutputConfigError(config: EvaluationOutputConfig, require
export const EVALUATION_NUMERIC_GRADED_HOGQL = `properties.$ai_evaluation_result_type = 'numeric' AND properties.$ai_evaluation_numeric_result IS NOT NULL AND (isNull(properties.$ai_evaluation_applicable) OR properties.$ai_evaluation_applicable != 'false') AND ${EVALUATION_NOT_SKIPPED_HOGQL}`
export const EVALUATION_BOOLEAN_GRADED_HOGQL = `(isNull(properties.$ai_evaluation_result_type) OR properties.$ai_evaluation_result_type = 'boolean') AND properties.$ai_evaluation_result IS NOT NULL AND ${EVALUATION_NOT_SKIPPED_HOGQL}`
export const EVALUATION_NUMERIC_MEAN_HOGQL = `avgIf(toFloat(properties.$ai_evaluation_numeric_result), ${EVALUATION_NUMERIC_GRADED_HOGQL})`
+// A skipped sentiment run has no label, so it leaves both sides of the ratio.
+export const EVALUATION_SENTIMENT_GRADED_HOGQL = `lower(properties.$ai_sentiment_label) IN ('positive', 'neutral', 'negative')`
export function evaluationGradedHogQL(outputType: string): string {
return outputType === 'categorical'
? EVALUATION_CATEGORICAL_GRADED_HOGQL
: outputType === 'numeric'
? EVALUATION_NUMERIC_GRADED_HOGQL
- : EVALUATION_BOOLEAN_GRADED_HOGQL
+ : outputType === 'sentiment'
+ ? EVALUATION_SENTIMENT_GRADED_HOGQL
+ : EVALUATION_BOOLEAN_GRADED_HOGQL
}
export function numericEvaluationPassedHogQL(evaluation: Pick<EvaluationConfig, 'output_config'>): string {
diff --git a/products/ai_observability/frontend/evaluations/llmEvaluationLogic.test.ts b/products/ai_observability/frontend/evaluations/llmEvaluationLogic.test.ts
index f3553e77ca71..a845be03ad44 100644
--- a/products/ai_observability/frontend/evaluations/llmEvaluationLogic.test.ts
+++ b/products/ai_observability/frontend/evaluations/llmEvaluationLogic.test.ts
@@ -1152,6 +1152,20 @@ return result`,
expect(logic.values.isReportableEvaluation).toBe(false)
})
+ it('charts the negative rate for a sentiment evaluation', () => {
+ logic = llmEvaluationLogic({ evaluationId: 'eval-123' })
+ logic.mount()
+ logic.actions.loadEvaluationSuccess({
+ ...mockEvaluation,
+ evaluation_type: 'sentiment',
+ evaluation_config: { source: 'user_messages' },
+ output_type: 'sentiment',
+ output_config: {},
+ model_configuration: null,
+ })
+ expect(logic.values.trendInsightUrl).toContain('Negative%20rate')
+ })
+
it.each([
{ trueCount: 999, expected: 99.9 },
{ trueCount: 1000, expected: 100 },
diff --git a/products/ai_observability/frontend/evaluations/llmEvaluationLogic.ts b/products/ai_observability/frontend/evaluations/llmEvaluationLogic.ts
index e798e7ee913e..120e4aabcdb9 100644
--- a/products/ai_observability/frontend/evaluations/llmEvaluationLogic.ts
+++ b/products/ai_observability/frontend/evaluations/llmEvaluationLogic.ts
@@ -36,6 +36,7 @@ import { evaluationErrorMessage } from './apiErrors'
import {
EVALUATION_NUMERIC_GRADED_HOGQL,
EVALUATION_NUMERIC_MEAN_HOGQL,
+ EVALUATION_SENTIMENT_GRADED_HOGQL,
numericEvaluationPassedHogQL,
evaluationPassedHogQL,
evaluationPassRateHogQL,
@@ -1650,12 +1651,15 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([
originalEvaluation: EvaluationConfig | null
): boolean => evaluationSupportsReports(isNewEvaluation ? evaluation : originalEvaluation),
],
- // The evaluation's headline result as a trends query: the pass rate, or the mean score without a passing rule.
+ // The evaluation's headline result as a trends query: the pass rate, the mean score without a passing rule, or
+ // the negative sentiment rate.
trendsQuery: [
(s) => [s.originalEvaluation],
(evaluation: EvaluationConfig | null): TrendsQuery | null =>
evaluation &&
- (evaluationSupportsRunOutcomes(evaluation) || evaluation.output_type === 'numeric') &&
+ (evaluationSupportsRunOutcomes(evaluation) ||
+ evaluation.output_type === 'numeric' ||
+ evaluation.output_type === 'sentiment') &&
evaluation.id
? ({
kind: NodeKind.TrendsQuery,
@@ -1663,22 +1667,27 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([
{
kind: NodeKind.EventsNode,
event: '$ai_evaluation',
- custom_name: `${evaluation.name} - ${evaluation.output_type === 'numeric' && !evaluation.output_config.passing_rule ? 'Mean score' : 'Pass rate'}`,
+ custom_name: `${evaluation.name} - ${evaluation.output_type === 'sentiment' ? 'Negative rate' : evaluation.output_type === 'numeric' && !evaluation.output_config.passing_rule ? 'Mean score' : 'Pass rate'}`,
math: HogQLMathType.HogQL,
math_hogql:
- evaluation.output_type === 'categorical'
+ evaluation.output_type === 'sentiment'
? evaluationPassRateHogQL(
- categoricalEvaluationPassedHogQL(evaluation),
- EVALUATION_CATEGORICAL_GRADED_HOGQL
+ `lower(properties.$ai_sentiment_label) = 'negative'`,
+ EVALUATION_SENTIMENT_GRADED_HOGQL
)
- : evaluation.output_type === 'numeric'
- ? evaluation.output_config.passing_rule
- ? evaluationPassRateHogQL(
- numericEvaluationPassedHogQL(evaluation),
- EVALUATION_NUMERIC_GRADED_HOGQL
- )
- : EVALUATION_NUMERIC_MEAN_HOGQL
- : evaluationPassRateHogQL(evaluationPassedHogQL(evaluation)),
+ : evaluation.output_type === 'categorical'
+ ? evaluationPassRateHogQL(
+ categoricalEvaluationPassedHogQL(evaluation),
+ EVALUATION_CATEGORICAL_GRADED_HOGQL
+ )
+ : evaluation.output_type === 'numeric'
+ ? evaluation.output_config.passing_rule
+ ? evaluationPassRateHogQL(
+ numericEvaluationPassedHogQL(evaluation),
+ EVALUATION_NUMERIC_GRADED_HOGQL
+ )
+ : EVALUATION_NUMERIC_MEAN_HOGQL
+ : evaluationPassRateHogQL(evaluationPassedHogQL(evaluation)),
properties: [
{
key: '$ai_evaluation_id',