:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(aio): stop retrying eval report Slack config errors (#124642)
Carlos Marchal326f70019419
diff --git a/posthog/templates/email/ai_observability_evaluation_report_disabled.html b/posthog/templates/email/ai_observability_evaluation_report_disabled.htmlnew file mode 100644index 000000000000..df032adbd879--- /dev/null+++ b/posthog/templates/email/ai_observability_evaluation_report_disabled.html@@ -0,0 +1,35 @@+{% extends "email/base.html" %}+{% block preheader %}Your report for "{{ evaluation_name }}" has been disabled{% endblock %}+{% block heading %}Your evaluation report has been disabled{% endblock %}+{% block section %}+ <p>+ PostHog automatically disabled the report for <strong>{{ evaluation_name }}</strong> because none of its destinations could receive it:+ </p>+ <div style="padding: 12px;+ background-color: #f8f8f8;+ border-left: 4px solid #e5484d;+ border-radius: 4px">+ {% for error in delivery_errors %}<p>{{ error }}</p>{% endfor %}+ </div>+ <p>+ Fix the destinations, then turn the report back on from the evaluation in+ <a href="{% absolute_uri evaluation_url %}">AI observability</a>.+ </p>+ <div class="mb mt text-center">+ <a class="button" href="{% absolute_uri evaluation_url %}">View evaluation</a>+ </div>+ <div class="mt">+ <p>+ If the button above doesn't work, paste this link into your browser:+ <br />+ <a href="{% absolute_uri evaluation_url %}">{% absolute_uri evaluation_url %}</a>+ </p>+ </div>+{% endblock %}+{% block footer %}+ Need help?+ <a href="https://posthog.com/questions?{{ utm_tags }}" target="_blank"><b>Visit support</b></a>+ or+ <a href="https://posthog.com/docs/ai-engineering?{{ utm_tags }}"+ target="_blank"><b>read our documentation</b></a>.+{% endblock %}diff --git a/posthog/temporal/ai_observability/eval_reports/auto_disable.py b/posthog/temporal/ai_observability/eval_reports/auto_disable.pynew file mode 100644index 000000000000..b763c12e1c94--- /dev/null+++ b/posthog/temporal/ai_observability/eval_reports/auto_disable.py@@ -0,0 +1,115 @@+from __future__ import annotations++from typing import TYPE_CHECKING++import structlog++from posthog.exceptions_capture import capture_exception++if TYPE_CHECKING:+ from posthog.models import User++ from products.ai_observability.backend.models.evaluation_reports import EvaluationReport, EvaluationReportRun++logger = structlog.get_logger(__name__)+++def disable_undeliverable_report(report: EvaluationReport, report_run: EvaluationReportRun) -> None:+ """Disable a report that no delivery target can receive, so it stops generating, and tell its owner."""+ from products.ai_observability.backend.models.evaluation_reports import EvaluationReport++ # Compare-and-swap so that only one of two racing runs sends the notifications. Matching the targets+ # this run tried keeps a report enabled when a user changes its targets while the delivery runs.+ if not EvaluationReport.objects.filter(+ pk=report.pk, team_id=report.team_id, enabled=True, delivery_targets=report.delivery_targets+ ).update(enabled=False):+ return+ report.enabled = False+ logger.warning("llma_eval_reports_auto_disabled", report_id=str(report.pk), team_id=report.team_id)++ recipient = _notification_recipient(report)+ if recipient is None:+ return++ for notify in (_create_in_app_notification, _send_email):+ # Disabling is the durable outcome. A failed notification must not fail the delivery activity.+ try:+ notify(report, report_run, recipient)+ except Exception as e:+ capture_exception(e)+ logger.warning(+ "llma_eval_reports_auto_disabled_notification_failed",+ report_id=str(report.pk),+ notifier=notify.__name__,+ exc_info=True,+ )+++def _notification_recipient(report: EvaluationReport) -> User | None:+ from products.access_control.backend.facade.user_access_control import UserAccessControl++ # An evaluation creates its default report without a creator, and configuring the report does not set one.+ recipient = report.created_by or report.evaluation.created_by+ if recipient is None:+ return None+ if not report.team.all_users_with_access().filter(id=recipient.id).exists():+ return None+ if not UserAccessControl(recipient, report.team).check_access_level_for_object(report.evaluation, "viewer"):+ return None+ return recipient+++def _evaluation_path(report: EvaluationReport) -> str:+ return f"/project/{report.team_id}/ai-evals/evaluations/{report.evaluation_id}"+++def _create_in_app_notification(report: EvaluationReport, report_run: EvaluationReportRun, recipient: User) -> None:+ from products.notifications.backend.facade.api import (+ NotificationData,+ NotificationType,+ Priority,+ TargetType,+ create_notification,+ )++ create_notification(+ NotificationData(+ team_id=report.team_id,+ notification_type=NotificationType.PIPELINE_FAILURE,+ priority=Priority.NORMAL,+ title=f"Report for {report.evaluation.name[:75]} was automatically disabled",+ body=(+ "PostHog automatically disabled this report because none of its destinations could receive it. "+ f"{' '.join(report_run.delivery_errors)} "+ "Turn the report back on after you fix this."+ ),+ target_type=TargetType.USER,+ target_id=str(recipient.id),+ resource_type="evaluation",+ resource_id=str(report.evaluation_id),+ source_url=_evaluation_path(report),+ idempotency_key=f"llma_eval_report_disabled_{report_run.pk}",+ )+ )+++def _send_email(report: EvaluationReport, report_run: EvaluationReportRun, recipient: User) -> None:+ from posthog.email import EmailMessage, is_email_available, single_line++ if not is_email_available(with_absolute_urls=True):+ return++ evaluation_name = report.evaluation.name+ message = EmailMessage(+ # Keyed on the run so that a retried activity does not send the email twice.+ campaign_key=f"llma_eval_report_disabled_{report.pk}_{report_run.pk}",+ subject=f'Your report for "{single_line(evaluation_name)}" has been automatically disabled',+ template_name="ai_observability_evaluation_report_disabled",+ template_context={+ "evaluation_name": evaluation_name,+ "delivery_errors": report_run.delivery_errors,+ "evaluation_url": _evaluation_path(report),+ },+ )+ message.add_user_recipient(recipient)+ message.send()diff --git a/posthog/temporal/ai_observability/eval_reports/delivery.py b/posthog/temporal/ai_observability/eval_reports/delivery.pyindex 2c1b4af5d2dc..ee487eba9206 100644--- a/posthog/temporal/ai_observability/eval_reports/delivery.py+++ b/posthog/temporal/ai_observability/eval_reports/delivery.py@@ -13,9 +13,12 @@ from urllib.parse import quote, urlencodeimport structlogfrom markdown_it import MarkdownItfrom markdown_to_mrkdwn import SlackMarkdownConverter+from slack_sdk.errors import SlackApiError+from posthog.dataclasses import frozenfrom posthog.models.team import Teamfrom posthog.slack.formatting import channel_id_from_target+from posthog.temporal.ai_observability.eval_reports.auto_disable import disable_undeliverable_reportfrom posthog.temporal.ai_observability.eval_reports.report_agent.schema import (METRICS_UNAVAILABLE_MESSAGE,Citation,@@ -39,6 +42,22 @@ _slack_converter = SlackMarkdownConverter()_INVITE_UTM_TAGS = "utm_source=posthog&utm_campaign=eval_report&utm_medium=slack"+_SLACK_RECONNECT_HINT = "The Slack connection no longer works. Reconnect Slack in your project settings."+# Slack errors that no retry can fix, each with the step the user must take.+_SLACK_CONFIG_ERROR_HINTS = {+ "not_in_channel": "The PostHog app is not in this channel. Invite it with /invite @PostHog.",+ "is_archived": "This channel is archived. Pick a different channel for this report.",+ # Slack also returns channel_not_found for a private channel that the app is not a member of.+ "channel_not_found": (+ "Slack can't find this channel. If the channel is private, invite the PostHog app with /invite @PostHog. "+ "Otherwise, pick a different channel for this report."+ ),+ **dict.fromkeys(+ ("account_inactive", "invalid_auth", "token_revoked", "missing_scope", "not_allowed_token_type"),+ _SLACK_RECONNECT_HINT,+ ),+}+# Inline styles for email-safe HTML (many clients strip <style> blocks)_EMAIL_TABLE_STYLE = 'style="border-collapse: collapse; width: 100%; margin: 8px 0; font-size: 14px;"'_EMAIL_TH_STYLE = (@@ -339,6 +358,30 @@ def deliver_email_report(return errors+@frozen+class SlackDeliveryFailure:+ message: str+ retryable: bool+++def _slack_channel_label(target_channel: str) -> str:+ return target_channel.partition("|")[2] or target_channel+++def _slack_config_failure(target_channel: str, hint: str) -> SlackDeliveryFailure:+ error_msg = f"Failed to send Slack message to {_slack_channel_label(target_channel)}: {hint}"+ logger.warning(error_msg)+ return SlackDeliveryFailure(message=error_msg, retryable=False)+++def _slack_delivery_failure(target_channel: str, error: Exception) -> SlackDeliveryFailure:+ if isinstance(error, SlackApiError) and (hint := _SLACK_CONFIG_ERROR_HINTS.get(error.response.get("error"))):+ return _slack_config_failure(target_channel, hint)+ error_msg = f"Failed to send Slack message to {_slack_channel_label(target_channel)}: {error}"+ logger.exception(error_msg)+ return SlackDeliveryFailure(message=error_msg, retryable=True)++def deliver_slack_report(report_run,targets: list[dict],@@ -347,20 +390,20 @@ def deliver_slack_report(project_id: int,period_start: str,period_end: str,-) -> list[str]:- """Send report via Slack. Returns list of errors (empty if all succeeded).+) -> list[SlackDeliveryFailure]:+ """Send report via Slack. Returns one failure per failed target (empty if all succeeded).Main message = agent title (Block Kit header) + metrics context + first section.Thread replies = sections[1:]. If there's only one section, no thread replies."""- from posthog.integrations import IntegrationManager+ from posthog.integrations import IntegrationManager, IntegrationNotFoundfrom posthog.models.integration import SlackIntegrationfrom products.slack_app.backend.facade.api import slack_followup_invitecontent = EvalReportContent.from_dict(report_run.content)citation_map = _build_citation_map(content.citations)- errors: list[str] = []+ failures: list[SlackDeliveryFailure] = []header_text = f"{evaluation_name}: {content.title}" if content.title else f"Evaluation report: {evaluation_name}"# Slack header blocks are limited to 150 chars, enforce that here.@@ -388,7 +431,7 @@ def deliver_slack_report(if not channel_id:error_msg = f"Failed to send Slack message to {channel}: no channel ID in target value"logger.warning(error_msg)- errors.append(error_msg)+ failures.append(SlackDeliveryFailure(message=error_msg, retryable=False))continuetry:@@ -440,12 +483,13 @@ def deliver_slack_report(text=mrkdwn_text[:3000],)+ except IntegrationNotFound:+ # delivery_targets is a JSON field, so a deleted Slack integration leaves its id behind.+ failures.append(_slack_config_failure(channel, _SLACK_RECONNECT_HINT))except Exception as e:- error_msg = f"Failed to send Slack message to {channel}: {e}"- logger.exception(error_msg)- errors.append(error_msg)+ failures.append(_slack_delivery_failure(channel, e))- return errors+ return failuresdef deliver_report(report_id: str, report_run_id: str) -> None:@@ -454,6 +498,8 @@ def deliver_report(report_id: str, report_run_id: str) -> None:Raises RuntimeError after persisting FAILED status if ALL delivery targetsfail, so the Temporal activity surfaces the failure and the retry policykicks in. Partial failures return normally (with delivery_status=partial_failure).+ If every failure is a Slack configuration error, a retry cannot fix it, so the+ report is disabled and its creator notified instead."""from products.ai_observability.backend.models.evaluation_reports import EvaluationReport, EvaluationReportRun@@ -479,12 +525,12 @@ def deliver_report(report_id: str, report_run_id: str) -> None:all_errors.extend(email_errors)slack_targets = [t for t in targets if t.get("type") == "slack"]- slack_errors: list[str] = []+ slack_failures: list[SlackDeliveryFailure] = []if slack_targets:- slack_errors = deliver_slack_report(+ slack_failures = deliver_slack_report(report_run, slack_targets, evaluation_name, team_id, project_id, period_start, period_end)- all_errors.extend(slack_errors)+ all_errors.extend(failure.message for failure in slack_failures)had_any_target = bool(email_targets or slack_targets)# Each email target may contain multiple comma-separated addresses, and deliver_email_report@@ -502,9 +548,9 @@ def deliver_report(report_id: str, report_run_id: str) -> None:increment_delivery("email", "delivered")for _ in range(len(email_errors)):increment_delivery("email", "failed")- for _ in range(slack_attempts - len(slack_errors)):+ for _ in range(slack_attempts - len(slack_failures)):increment_delivery("slack", "delivered")- for _ in range(len(slack_errors)):+ for _ in range(len(slack_failures)):increment_delivery("slack", "failed")if not had_any_target:@@ -530,6 +576,12 @@ def deliver_report(report_id: str, report_run_id: str) -> None:error_count=len(all_errors),)- if all_failed:+ if not all_failed:+ return++ retryable = bool(email_errors) or any(failure.retryable for failure in slack_failures)+ if retryable:# Raise so the Temporal activity fails and retries fire per DELIVER_RETRY_POLICY.raise RuntimeError(f"All delivery targets failed for report run {report_run_id}: {'; '.join(all_errors)}")++ disable_undeliverable_report(report, report_run)diff --git a/posthog/temporal/ai_observability/eval_reports/test/test_auto_disable.py b/posthog/temporal/ai_observability/eval_reports/test/test_auto_disable.pynew file mode 100644index 000000000000..8f5faf27e33b--- /dev/null+++ b/posthog/temporal/ai_observability/eval_reports/test/test_auto_disable.py@@ -0,0 +1,88 @@+from posthog.test.base import BaseTest+from unittest.mock import MagicMock, patch++from django.utils import timezone++from parameterized import parameterized++from posthog.models import Organization, User+from posthog.temporal.ai_observability.eval_reports.auto_disable import disable_undeliverable_report++from products.ai_observability.backend.models.evaluation_reports import EvaluationReport+from products.ai_observability.backend.models.evaluations import Evaluation+++@patch("posthog.email.is_email_available", return_value=True)+@patch("posthog.email.EmailMessage")+@patch("products.notifications.backend.facade.api.create_notification")+class TestDisableUndeliverableReport(BaseTest):+ def _create_report(self, created_by: User | None) -> EvaluationReport:+ evaluation = Evaluation.objects.create(+ team=self.team,+ name="Test\nEval",+ evaluation_type="llm_judge",+ evaluation_config={"prompt": "test prompt"},+ output_type="boolean",+ output_config={},+ enabled=True,+ created_by=self.user,+ conditions=[{"id": "c1", "rollout_percentage": 100, "properties": []}],+ )+ return EvaluationReport.objects.create(+ team=self.team,+ evaluation=evaluation,+ frequency=EvaluationReport.Frequency.SCHEDULED,+ rrule="FREQ=DAILY",+ starts_at=timezone.now(),+ delivery_targets=[{"type": "slack", "integration_id": 1, "channel": "C0123|#reports"}],+ created_by=created_by,+ )++ def _make_report_run(self) -> MagicMock:+ return MagicMock(pk="run-id", delivery_errors=["Failed to send Slack message to #reports: Invite it."])++ @parameterized.expand([("report_creator", True), ("evaluation_creator_of_default_report", False)])+ def test_disables_once_and_notifies_owner(+ self, mock_create_notification, mock_email_message, _mock_available, _name, report_has_creator+ ):+ report = self._create_report(created_by=self.user if report_has_creator else None)++ disable_undeliverable_report(report, self._make_report_run())+ disable_undeliverable_report(EvaluationReport.objects.get(pk=report.pk), self._make_report_run())++ report.refresh_from_db()+ self.assertFalse(report.enabled)+ mock_create_notification.assert_called_once()+ self.assertEqual(mock_create_notification.call_args.args[0].target_id, str(self.user.id))+ mock_email_message.return_value.add_user_recipient.assert_called_once_with(self.user)+ mock_email_message.return_value.send.assert_called_once()+ self.assertNotIn("\n", mock_email_message.call_args.kwargs["subject"])++ def test_report_with_changed_targets_stays_enabled(+ self, mock_create_notification, mock_email_message, _mock_available+ ):+ report = self._create_report(created_by=self.user)+ EvaluationReport.objects.filter(pk=report.pk).update(+ delivery_targets=[{"type": "slack", "integration_id": 1, "channel": "C0456|#fixed"}]+ )++ disable_undeliverable_report(report, self._make_report_run())++ report.refresh_from_db()+ self.assertTrue(report.enabled)+ mock_create_notification.assert_not_called()+ mock_email_message.assert_not_called()++ def test_creator_outside_the_project_is_not_notified(+ self, mock_create_notification, mock_email_message, _mock_available+ ):+ other_org = Organization.objects.create(name="Other org")+ outsider = User.objects.create_and_join(other_org, "[email protected]", "password")+ report = self._create_report(created_by=outsider)++ disable_undeliverable_report(report, self._make_report_run())++ report.refresh_from_db()+ self.assertFalse(report.enabled)+ mock_create_notification.assert_not_called()+ mock_email_message.assert_not_called()diff --git a/posthog/temporal/ai_observability/eval_reports/test/test_delivery.py b/posthog/temporal/ai_observability/eval_reports/test/test_delivery.pyindex 2ce62e1c5cbb..099c0fe93250 100644--- a/posthog/temporal/ai_observability/eval_reports/test/test_delivery.py+++ b/posthog/temporal/ai_observability/eval_reports/test/test_delivery.py@@ -1,11 +1,16 @@+from contextlib import nullcontext+from unittest.mock import MagicMock, patchfrom django.test import SimpleTestCasefrom parameterized import parameterized+from slack_sdk.errors import SlackApiError+from posthog.integrations import IntegrationNotFoundfrom posthog.temporal.ai_observability.eval_reports.delivery import (CitationMap,+ SlackDeliveryFailure,_build_citation_map,_format_period_for_display,_inline_email_styles,@@ -487,12 +492,35 @@ class TestDeliverReport(SimpleTestCase):mock_report_qs.select_related.return_value.get.return_value = reportmock_run_qs.get.return_value = runmock_email.return_value = []- mock_slack.return_value = ["slack failed"]+ mock_slack.return_value = [SlackDeliveryFailure(message="slack failed", retryable=True)]deliver_report("report-id", "run-id")self.assertEqual(run.delivery_status, "partial_failure")+ @parameterized.expand([("retryable_raises_for_retry", True), ("config_error_disables_report", False)])+ @patch("posthog.temporal.ai_observability.eval_reports.delivery.disable_undeliverable_report")+ @patch("posthog.temporal.ai_observability.eval_reports.delivery.deliver_email_report")+ @patch("posthog.temporal.ai_observability.eval_reports.delivery.deliver_slack_report")+ @patch("products.ai_observability.backend.models.evaluation_reports.EvaluationReportRun.objects")+ @patch("products.ai_observability.backend.models.evaluation_reports.EvaluationReport.objects")+ def test_all_slack_targets_failed(+ self, _name, retryable, mock_report_qs, mock_run_qs, mock_slack, _mock_email, mock_disable+ ):+ report = self._make_report([{"type": "slack", "integration_id": 1, "channel": "C0123|#reports"}])+ run = self._make_report_run()++ mock_report_qs.select_related.return_value.get.return_value = report+ mock_run_qs.get.return_value = run+ mock_slack.return_value = [SlackDeliveryFailure(message="slack failed", retryable=retryable)]++ with self.assertRaises(RuntimeError) if retryable else nullcontext():+ deliver_report("report-id", "run-id")++ self.assertEqual(run.delivery_status, "failed")+ self.assertEqual(run.delivery_errors, ["slack failed"])+ self.assertEqual(mock_disable.called, not retryable)+@patch("posthog.temporal.ai_observability.eval_reports.delivery.deliver_email_report")@patch("posthog.temporal.ai_observability.eval_reports.delivery.deliver_slack_report")@patch("products.ai_observability.backend.models.evaluation_reports.EvaluationReportRun.objects")@@ -678,5 +706,47 @@ class TestDeliverSlackReport(SimpleTestCase):)self.assertEqual(len(errors), 1)- self.assertIn("|#tech-devops-cron", errors[0])+ self.assertIn("|#tech-devops-cron", errors[0].message)+ self.assertFalse(errors[0].retryable)client.chat_postMessage.assert_not_called()++ @parameterized.expand(+ [+ ("bot_not_in_channel", True, "not_in_channel", False, "#evals: The PostHog app is not in this channel"),+ ("revoked_token", True, "token_revoked", False, "Reconnect Slack"),+ ("rate_limited", True, "ratelimited", True, "ratelimited"),+ ("integration_deleted", False, None, False, "Reconnect Slack"),+ ]+ )+ @patch("posthog.models.integration.SlackIntegration")+ @patch("posthog.integrations.IntegrationManager")+ def test_slack_failure_is_classified(+ self,+ _name,+ integration_exists,+ slack_error,+ expected_retryable,+ expected_text,+ mock_integration,+ mock_slack_integration,+ ):+ if not integration_exists:+ mock_integration.slack.oauth.get.side_effect = IntegrationNotFound("deleted")+ client = MagicMock()+ client.chat_postMessage.side_effect = SlackApiError("failed", {"ok": False, "error": slack_error})+ mock_slack_integration.return_value.client = client++ targets = [{"type": "slack", "integration_id": 1, "channel": "C0B5CHB0JQH|#evals"}]+ errors = deliver_slack_report(+ self._make_report_run(),+ targets,+ evaluation_name="Test Eval",+ team_id=1,+ project_id=1,+ period_start="2026-03-01T00:00:00+00:00",+ period_end="2026-03-02T00:00:00+00:00",+ )++ self.assertEqual(len(errors), 1)+ self.assertEqual(errors[0].retryable, expected_retryable)+ self.assertIn(expected_text, errors[0].message)