:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(aio): add a trend insight for sentiment evaluations (#126196)
Bernat Torres4cb88ec63ec3
diff --git a/products/ai_observability/frontend/evaluations/constants.ts b/products/ai_observability/frontend/evaluations/constants.tsindex e271b0f620bb..48f9896f496d 100644--- a/products/ai_observability/frontend/evaluations/constants.ts+++ b/products/ai_observability/frontend/evaluations/constants.ts@@ -74,13 +74,17 @@ export function numericOutputConfigError(config: EvaluationOutputConfig, requireexport const EVALUATION_NUMERIC_GRADED_HOGQL = `properties.$ai_evaluation_result_type = 'numeric' AND properties.$ai_evaluation_numeric_result IS NOT NULL AND (isNull(properties.$ai_evaluation_applicable) OR properties.$ai_evaluation_applicable != 'false') AND ${EVALUATION_NOT_SKIPPED_HOGQL}`export const EVALUATION_BOOLEAN_GRADED_HOGQL = `(isNull(properties.$ai_evaluation_result_type) OR properties.$ai_evaluation_result_type = 'boolean') AND properties.$ai_evaluation_result IS NOT NULL AND ${EVALUATION_NOT_SKIPPED_HOGQL}`export const EVALUATION_NUMERIC_MEAN_HOGQL = `avgIf(toFloat(properties.$ai_evaluation_numeric_result), ${EVALUATION_NUMERIC_GRADED_HOGQL})`+// A skipped sentiment run has no label, so it leaves both sides of the ratio.+export const EVALUATION_SENTIMENT_GRADED_HOGQL = `lower(properties.$ai_sentiment_label) IN ('positive', 'neutral', 'negative')`export function evaluationGradedHogQL(outputType: string): string {return outputType === 'categorical'? EVALUATION_CATEGORICAL_GRADED_HOGQL: outputType === 'numeric'? EVALUATION_NUMERIC_GRADED_HOGQL- : EVALUATION_BOOLEAN_GRADED_HOGQL+ : outputType === 'sentiment'+ ? EVALUATION_SENTIMENT_GRADED_HOGQL+ : EVALUATION_BOOLEAN_GRADED_HOGQL}export function numericEvaluationPassedHogQL(evaluation: Pick<EvaluationConfig, 'output_config'>): string {diff --git a/products/ai_observability/frontend/evaluations/llmEvaluationLogic.test.ts b/products/ai_observability/frontend/evaluations/llmEvaluationLogic.test.tsindex f3553e77ca71..a845be03ad44 100644--- a/products/ai_observability/frontend/evaluations/llmEvaluationLogic.test.ts+++ b/products/ai_observability/frontend/evaluations/llmEvaluationLogic.test.ts@@ -1152,6 +1152,20 @@ return result`,expect(logic.values.isReportableEvaluation).toBe(false)})+ it('charts the negative rate for a sentiment evaluation', () => {+ logic = llmEvaluationLogic({ evaluationId: 'eval-123' })+ logic.mount()+ logic.actions.loadEvaluationSuccess({+ ...mockEvaluation,+ evaluation_type: 'sentiment',+ evaluation_config: { source: 'user_messages' },+ output_type: 'sentiment',+ output_config: {},+ model_configuration: null,+ })+ expect(logic.values.trendInsightUrl).toContain('Negative%20rate')+ })+it.each([{ trueCount: 999, expected: 99.9 },{ trueCount: 1000, expected: 100 },diff --git a/products/ai_observability/frontend/evaluations/llmEvaluationLogic.ts b/products/ai_observability/frontend/evaluations/llmEvaluationLogic.tsindex e798e7ee913e..120e4aabcdb9 100644--- a/products/ai_observability/frontend/evaluations/llmEvaluationLogic.ts+++ b/products/ai_observability/frontend/evaluations/llmEvaluationLogic.ts@@ -36,6 +36,7 @@ import { evaluationErrorMessage } from './apiErrors'import {EVALUATION_NUMERIC_GRADED_HOGQL,EVALUATION_NUMERIC_MEAN_HOGQL,+ EVALUATION_SENTIMENT_GRADED_HOGQL,numericEvaluationPassedHogQL,evaluationPassedHogQL,evaluationPassRateHogQL,@@ -1650,12 +1651,15 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([originalEvaluation: EvaluationConfig | null): boolean => evaluationSupportsReports(isNewEvaluation ? evaluation : originalEvaluation),],- // The evaluation's headline result as a trends query: the pass rate, or the mean score without a passing rule.+ // The evaluation's headline result as a trends query: the pass rate, the mean score without a passing rule, or+ // the negative sentiment rate.trendsQuery: [(s) => [s.originalEvaluation],(evaluation: EvaluationConfig | null): TrendsQuery | null =>evaluation &&- (evaluationSupportsRunOutcomes(evaluation) || evaluation.output_type === 'numeric') &&+ (evaluationSupportsRunOutcomes(evaluation) ||+ evaluation.output_type === 'numeric' ||+ evaluation.output_type === 'sentiment') &&evaluation.id? ({kind: NodeKind.TrendsQuery,@@ -1663,22 +1667,27 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([{kind: NodeKind.EventsNode,event: '$ai_evaluation',- custom_name: `${evaluation.name} - ${evaluation.output_type === 'numeric' && !evaluation.output_config.passing_rule ? 'Mean score' : 'Pass rate'}`,+ custom_name: `${evaluation.name} - ${evaluation.output_type === 'sentiment' ? 'Negative rate' : evaluation.output_type === 'numeric' && !evaluation.output_config.passing_rule ? 'Mean score' : 'Pass rate'}`,math: HogQLMathType.HogQL,math_hogql:- evaluation.output_type === 'categorical'+ evaluation.output_type === 'sentiment'? evaluationPassRateHogQL(- categoricalEvaluationPassedHogQL(evaluation),- EVALUATION_CATEGORICAL_GRADED_HOGQL+ `lower(properties.$ai_sentiment_label) = 'negative'`,+ EVALUATION_SENTIMENT_GRADED_HOGQL)- : evaluation.output_type === 'numeric'- ? evaluation.output_config.passing_rule- ? evaluationPassRateHogQL(- numericEvaluationPassedHogQL(evaluation),- EVALUATION_NUMERIC_GRADED_HOGQL- )- : EVALUATION_NUMERIC_MEAN_HOGQL- : evaluationPassRateHogQL(evaluationPassedHogQL(evaluation)),+ : evaluation.output_type === 'categorical'+ ? evaluationPassRateHogQL(+ categoricalEvaluationPassedHogQL(evaluation),+ EVALUATION_CATEGORICAL_GRADED_HOGQL+ )+ : evaluation.output_type === 'numeric'+ ? evaluation.output_config.passing_rule+ ? evaluationPassRateHogQL(+ numericEvaluationPassedHogQL(evaluation),+ EVALUATION_NUMERIC_GRADED_HOGQL+ )+ : EVALUATION_NUMERIC_MEAN_HOGQL+ : evaluationPassRateHogQL(evaluationPassedHogQL(evaluation)),properties: [{key: '$ai_evaluation_id',