Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(aio): stop retrying eval report Slack config errors (#124642)

Carlos Marchal326f70019419
diff --git a/posthog/templates/email/ai_observability_evaluation_report_disabled.html b/posthog/templates/email/ai_observability_evaluation_report_disabled.html
new file mode 100644
index 000000000000..df032adbd879
--- /dev/null
+++ b/posthog/templates/email/ai_observability_evaluation_report_disabled.html
@@ -0,0 +1,35 @@
+{% extends "email/base.html" %}
+{% block preheader %}Your report for "{{ evaluation_name }}" has been disabled{% endblock %}
+{% block heading %}Your evaluation report has been disabled{% endblock %}
+{% block section %}
+ <p>
+ PostHog automatically disabled the report for <strong>{{ evaluation_name }}</strong> because none of its destinations could receive it:
+ </p>
+ <div style="padding: 12px;
+ background-color: #f8f8f8;
+ border-left: 4px solid #e5484d;
+ border-radius: 4px">
+ {% for error in delivery_errors %}<p>{{ error }}</p>{% endfor %}
+ </div>
+ <p>
+ Fix the destinations, then turn the report back on from the evaluation in
+ <a href="{% absolute_uri evaluation_url %}">AI observability</a>.
+ </p>
+ <div class="mb mt text-center">
+ <a class="button" href="{% absolute_uri evaluation_url %}">View evaluation</a>
+ </div>
+ <div class="mt">
+ <p>
+ If the button above doesn't work, paste this link into your browser:
+ <br />
+ <a href="{% absolute_uri evaluation_url %}">{% absolute_uri evaluation_url %}</a>
+ </p>
+ </div>
+{% endblock %}
+{% block footer %}
+ Need help?
+ <a href="https://posthog.com/questions?{{ utm_tags }}" target="_blank"><b>Visit support</b></a>
+ or
+ <a href="https://posthog.com/docs/ai-engineering?{{ utm_tags }}"
+ target="_blank"><b>read our documentation</b></a>.
+{% endblock %}
diff --git a/posthog/temporal/ai_observability/eval_reports/auto_disable.py b/posthog/temporal/ai_observability/eval_reports/auto_disable.py
new file mode 100644
index 000000000000..b763c12e1c94
--- /dev/null
+++ b/posthog/temporal/ai_observability/eval_reports/auto_disable.py
@@ -0,0 +1,115 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import structlog
+
+from posthog.exceptions_capture import capture_exception
+
+if TYPE_CHECKING:
+ from posthog.models import User
+
+ from products.ai_observability.backend.models.evaluation_reports import EvaluationReport, EvaluationReportRun
+
+logger = structlog.get_logger(__name__)
+
+
+def disable_undeliverable_report(report: EvaluationReport, report_run: EvaluationReportRun) -> None:
+ """Disable a report that no delivery target can receive, so it stops generating, and tell its owner."""
+ from products.ai_observability.backend.models.evaluation_reports import EvaluationReport
+
+ # Compare-and-swap so that only one of two racing runs sends the notifications. Matching the targets
+ # this run tried keeps a report enabled when a user changes its targets while the delivery runs.
+ if not EvaluationReport.objects.filter(
+ pk=report.pk, team_id=report.team_id, enabled=True, delivery_targets=report.delivery_targets
+ ).update(enabled=False):
+ return
+ report.enabled = False
+ logger.warning("llma_eval_reports_auto_disabled", report_id=str(report.pk), team_id=report.team_id)
+
+ recipient = _notification_recipient(report)
+ if recipient is None:
+ return
+
+ for notify in (_create_in_app_notification, _send_email):
+ # Disabling is the durable outcome. A failed notification must not fail the delivery activity.
+ try:
+ notify(report, report_run, recipient)
+ except Exception as e:
+ capture_exception(e)
+ logger.warning(
+ "llma_eval_reports_auto_disabled_notification_failed",
+ report_id=str(report.pk),
+ notifier=notify.__name__,
+ exc_info=True,
+ )
+
+
+def _notification_recipient(report: EvaluationReport) -> User | None:
+ from products.access_control.backend.facade.user_access_control import UserAccessControl
+
+ # An evaluation creates its default report without a creator, and configuring the report does not set one.
+ recipient = report.created_by or report.evaluation.created_by
+ if recipient is None:
+ return None
+ if not report.team.all_users_with_access().filter(id=recipient.id).exists():
+ return None
+ if not UserAccessControl(recipient, report.team).check_access_level_for_object(report.evaluation, "viewer"):
+ return None
+ return recipient
+
+
+def _evaluation_path(report: EvaluationReport) -> str:
+ return f"/project/{report.team_id}/ai-evals/evaluations/{report.evaluation_id}"
+
+
+def _create_in_app_notification(report: EvaluationReport, report_run: EvaluationReportRun, recipient: User) -> None:
+ from products.notifications.backend.facade.api import (
+ NotificationData,
+ NotificationType,
+ Priority,
+ TargetType,
+ create_notification,
+ )
+
+ create_notification(
+ NotificationData(
+ team_id=report.team_id,
+ notification_type=NotificationType.PIPELINE_FAILURE,
+ priority=Priority.NORMAL,
+ title=f"Report for {report.evaluation.name[:75]} was automatically disabled",
+ body=(
+ "PostHog automatically disabled this report because none of its destinations could receive it. "
+ f"{' '.join(report_run.delivery_errors)} "
+ "Turn the report back on after you fix this."
+ ),
+ target_type=TargetType.USER,
+ target_id=str(recipient.id),
+ resource_type="evaluation",
+ resource_id=str(report.evaluation_id),
+ source_url=_evaluation_path(report),
+ idempotency_key=f"llma_eval_report_disabled_{report_run.pk}",
+ )
+ )
+
+
+def _send_email(report: EvaluationReport, report_run: EvaluationReportRun, recipient: User) -> None:
+ from posthog.email import EmailMessage, is_email_available, single_line
+
+ if not is_email_available(with_absolute_urls=True):
+ return
+
+ evaluation_name = report.evaluation.name
+ message = EmailMessage(
+ # Keyed on the run so that a retried activity does not send the email twice.
+ campaign_key=f"llma_eval_report_disabled_{report.pk}_{report_run.pk}",
+ subject=f'Your report for "{single_line(evaluation_name)}" has been automatically disabled',
+ template_name="ai_observability_evaluation_report_disabled",
+ template_context={
+ "evaluation_name": evaluation_name,
+ "delivery_errors": report_run.delivery_errors,
+ "evaluation_url": _evaluation_path(report),
+ },
+ )
+ message.add_user_recipient(recipient)
+ message.send()
diff --git a/posthog/temporal/ai_observability/eval_reports/delivery.py b/posthog/temporal/ai_observability/eval_reports/delivery.py
index 2c1b4af5d2dc..ee487eba9206 100644
--- a/posthog/temporal/ai_observability/eval_reports/delivery.py
+++ b/posthog/temporal/ai_observability/eval_reports/delivery.py
@@ -13,9 +13,12 @@ from urllib.parse import quote, urlencode
import structlog
from markdown_it import MarkdownIt
from markdown_to_mrkdwn import SlackMarkdownConverter
+from slack_sdk.errors import SlackApiError
+from posthog.dataclasses import frozen
from posthog.models.team import Team
from posthog.slack.formatting import channel_id_from_target
+from posthog.temporal.ai_observability.eval_reports.auto_disable import disable_undeliverable_report
from posthog.temporal.ai_observability.eval_reports.report_agent.schema import (
METRICS_UNAVAILABLE_MESSAGE,
Citation,
@@ -39,6 +42,22 @@ _slack_converter = SlackMarkdownConverter()
_INVITE_UTM_TAGS = "utm_source=posthog&utm_campaign=eval_report&utm_medium=slack"
+_SLACK_RECONNECT_HINT = "The Slack connection no longer works. Reconnect Slack in your project settings."
+# Slack errors that no retry can fix, each with the step the user must take.
+_SLACK_CONFIG_ERROR_HINTS = {
+ "not_in_channel": "The PostHog app is not in this channel. Invite it with /invite @PostHog.",
+ "is_archived": "This channel is archived. Pick a different channel for this report.",
+ # Slack also returns channel_not_found for a private channel that the app is not a member of.
+ "channel_not_found": (
+ "Slack can't find this channel. If the channel is private, invite the PostHog app with /invite @PostHog. "
+ "Otherwise, pick a different channel for this report."
+ ),
+ **dict.fromkeys(
+ ("account_inactive", "invalid_auth", "token_revoked", "missing_scope", "not_allowed_token_type"),
+ _SLACK_RECONNECT_HINT,
+ ),
+}
+
# Inline styles for email-safe HTML (many clients strip <style> blocks)
_EMAIL_TABLE_STYLE = 'style="border-collapse: collapse; width: 100%; margin: 8px 0; font-size: 14px;"'
_EMAIL_TH_STYLE = (
@@ -339,6 +358,30 @@ def deliver_email_report(
return errors
+@frozen
+class SlackDeliveryFailure:
+ message: str
+ retryable: bool
+
+
+def _slack_channel_label(target_channel: str) -> str:
+ return target_channel.partition("|")[2] or target_channel
+
+
+def _slack_config_failure(target_channel: str, hint: str) -> SlackDeliveryFailure:
+ error_msg = f"Failed to send Slack message to {_slack_channel_label(target_channel)}: {hint}"
+ logger.warning(error_msg)
+ return SlackDeliveryFailure(message=error_msg, retryable=False)
+
+
+def _slack_delivery_failure(target_channel: str, error: Exception) -> SlackDeliveryFailure:
+ if isinstance(error, SlackApiError) and (hint := _SLACK_CONFIG_ERROR_HINTS.get(error.response.get("error"))):
+ return _slack_config_failure(target_channel, hint)
+ error_msg = f"Failed to send Slack message to {_slack_channel_label(target_channel)}: {error}"
+ logger.exception(error_msg)
+ return SlackDeliveryFailure(message=error_msg, retryable=True)
+
+
def deliver_slack_report(
report_run,
targets: list[dict],
@@ -347,20 +390,20 @@ def deliver_slack_report(
project_id: int,
period_start: str,
period_end: str,
-) -> list[str]:
- """Send report via Slack. Returns list of errors (empty if all succeeded).
+) -> list[SlackDeliveryFailure]:
+ """Send report via Slack. Returns one failure per failed target (empty if all succeeded).
Main message = agent title (Block Kit header) + metrics context + first section.
Thread replies = sections[1:]. If there's only one section, no thread replies.
"""
- from posthog.integrations import IntegrationManager
+ from posthog.integrations import IntegrationManager, IntegrationNotFound
from posthog.models.integration import SlackIntegration
from products.slack_app.backend.facade.api import slack_followup_invite
content = EvalReportContent.from_dict(report_run.content)
citation_map = _build_citation_map(content.citations)
- errors: list[str] = []
+ failures: list[SlackDeliveryFailure] = []
header_text = f"{evaluation_name}: {content.title}" if content.title else f"Evaluation report: {evaluation_name}"
# Slack header blocks are limited to 150 chars, enforce that here.
@@ -388,7 +431,7 @@ def deliver_slack_report(
if not channel_id:
error_msg = f"Failed to send Slack message to {channel}: no channel ID in target value"
logger.warning(error_msg)
- errors.append(error_msg)
+ failures.append(SlackDeliveryFailure(message=error_msg, retryable=False))
continue
try:
@@ -440,12 +483,13 @@ def deliver_slack_report(
text=mrkdwn_text[:3000],
)
+ except IntegrationNotFound:
+ # delivery_targets is a JSON field, so a deleted Slack integration leaves its id behind.
+ failures.append(_slack_config_failure(channel, _SLACK_RECONNECT_HINT))
except Exception as e:
- error_msg = f"Failed to send Slack message to {channel}: {e}"
- logger.exception(error_msg)
- errors.append(error_msg)
+ failures.append(_slack_delivery_failure(channel, e))
- return errors
+ return failures
def deliver_report(report_id: str, report_run_id: str) -> None:
@@ -454,6 +498,8 @@ def deliver_report(report_id: str, report_run_id: str) -> None:
Raises RuntimeError after persisting FAILED status if ALL delivery targets
fail, so the Temporal activity surfaces the failure and the retry policy
kicks in. Partial failures return normally (with delivery_status=partial_failure).
+ If every failure is a Slack configuration error, a retry cannot fix it, so the
+ report is disabled and its creator notified instead.
"""
from products.ai_observability.backend.models.evaluation_reports import EvaluationReport, EvaluationReportRun
@@ -479,12 +525,12 @@ def deliver_report(report_id: str, report_run_id: str) -> None:
all_errors.extend(email_errors)
slack_targets = [t for t in targets if t.get("type") == "slack"]
- slack_errors: list[str] = []
+ slack_failures: list[SlackDeliveryFailure] = []
if slack_targets:
- slack_errors = deliver_slack_report(
+ slack_failures = deliver_slack_report(
report_run, slack_targets, evaluation_name, team_id, project_id, period_start, period_end
)
- all_errors.extend(slack_errors)
+ all_errors.extend(failure.message for failure in slack_failures)
had_any_target = bool(email_targets or slack_targets)
# Each email target may contain multiple comma-separated addresses, and deliver_email_report
@@ -502,9 +548,9 @@ def deliver_report(report_id: str, report_run_id: str) -> None:
increment_delivery("email", "delivered")
for _ in range(len(email_errors)):
increment_delivery("email", "failed")
- for _ in range(slack_attempts - len(slack_errors)):
+ for _ in range(slack_attempts - len(slack_failures)):
increment_delivery("slack", "delivered")
- for _ in range(len(slack_errors)):
+ for _ in range(len(slack_failures)):
increment_delivery("slack", "failed")
if not had_any_target:
@@ -530,6 +576,12 @@ def deliver_report(report_id: str, report_run_id: str) -> None:
error_count=len(all_errors),
)
- if all_failed:
+ if not all_failed:
+ return
+
+ retryable = bool(email_errors) or any(failure.retryable for failure in slack_failures)
+ if retryable:
# Raise so the Temporal activity fails and retries fire per DELIVER_RETRY_POLICY.
raise RuntimeError(f"All delivery targets failed for report run {report_run_id}: {'; '.join(all_errors)}")
+
+ disable_undeliverable_report(report, report_run)
diff --git a/posthog/temporal/ai_observability/eval_reports/test/test_auto_disable.py b/posthog/temporal/ai_observability/eval_reports/test/test_auto_disable.py
new file mode 100644
index 000000000000..8f5faf27e33b
--- /dev/null
+++ b/posthog/temporal/ai_observability/eval_reports/test/test_auto_disable.py
@@ -0,0 +1,88 @@
+from posthog.test.base import BaseTest
+from unittest.mock import MagicMock, patch
+
+from django.utils import timezone
+
+from parameterized import parameterized
+
+from posthog.models import Organization, User
+from posthog.temporal.ai_observability.eval_reports.auto_disable import disable_undeliverable_report
+
+from products.ai_observability.backend.models.evaluation_reports import EvaluationReport
+from products.ai_observability.backend.models.evaluations import Evaluation
+
+
+@patch("posthog.email.is_email_available", return_value=True)
+@patch("posthog.email.EmailMessage")
+@patch("products.notifications.backend.facade.api.create_notification")
+class TestDisableUndeliverableReport(BaseTest):
+ def _create_report(self, created_by: User | None) -> EvaluationReport:
+ evaluation = Evaluation.objects.create(
+ team=self.team,
+ name="Test\nEval",
+ evaluation_type="llm_judge",
+ evaluation_config={"prompt": "test prompt"},
+ output_type="boolean",
+ output_config={},
+ enabled=True,
+ created_by=self.user,
+ conditions=[{"id": "c1", "rollout_percentage": 100, "properties": []}],
+ )
+ return EvaluationReport.objects.create(
+ team=self.team,
+ evaluation=evaluation,
+ frequency=EvaluationReport.Frequency.SCHEDULED,
+ rrule="FREQ=DAILY",
+ starts_at=timezone.now(),
+ delivery_targets=[{"type": "slack", "integration_id": 1, "channel": "C0123|#reports"}],
+ created_by=created_by,
+ )
+
+ def _make_report_run(self) -> MagicMock:
+ return MagicMock(pk="run-id", delivery_errors=["Failed to send Slack message to #reports: Invite it."])
+
+ @parameterized.expand([("report_creator", True), ("evaluation_creator_of_default_report", False)])
+ def test_disables_once_and_notifies_owner(
+ self, mock_create_notification, mock_email_message, _mock_available, _name, report_has_creator
+ ):
+ report = self._create_report(created_by=self.user if report_has_creator else None)
+
+ disable_undeliverable_report(report, self._make_report_run())
+ disable_undeliverable_report(EvaluationReport.objects.get(pk=report.pk), self._make_report_run())
+
+ report.refresh_from_db()
+ self.assertFalse(report.enabled)
+ mock_create_notification.assert_called_once()
+ self.assertEqual(mock_create_notification.call_args.args[0].target_id, str(self.user.id))
+ mock_email_message.return_value.add_user_recipient.assert_called_once_with(self.user)
+ mock_email_message.return_value.send.assert_called_once()
+ self.assertNotIn("\n", mock_email_message.call_args.kwargs["subject"])
+
+ def test_report_with_changed_targets_stays_enabled(
+ self, mock_create_notification, mock_email_message, _mock_available
+ ):
+ report = self._create_report(created_by=self.user)
+ EvaluationReport.objects.filter(pk=report.pk).update(
+ delivery_targets=[{"type": "slack", "integration_id": 1, "channel": "C0456|#fixed"}]
+ )
+
+ disable_undeliverable_report(report, self._make_report_run())
+
+ report.refresh_from_db()
+ self.assertTrue(report.enabled)
+ mock_create_notification.assert_not_called()
+ mock_email_message.assert_not_called()
+
+ def test_creator_outside_the_project_is_not_notified(
+ self, mock_create_notification, mock_email_message, _mock_available
+ ):
+ other_org = Organization.objects.create(name="Other org")
+ outsider = User.objects.create_and_join(other_org, "[email protected]", "password")
+ report = self._create_report(created_by=outsider)
+
+ disable_undeliverable_report(report, self._make_report_run())
+
+ report.refresh_from_db()
+ self.assertFalse(report.enabled)
+ mock_create_notification.assert_not_called()
+ mock_email_message.assert_not_called()
diff --git a/posthog/temporal/ai_observability/eval_reports/test/test_delivery.py b/posthog/temporal/ai_observability/eval_reports/test/test_delivery.py
index 2ce62e1c5cbb..099c0fe93250 100644
--- a/posthog/temporal/ai_observability/eval_reports/test/test_delivery.py
+++ b/posthog/temporal/ai_observability/eval_reports/test/test_delivery.py
@@ -1,11 +1,16 @@
+from contextlib import nullcontext
+
from unittest.mock import MagicMock, patch
from django.test import SimpleTestCase
from parameterized import parameterized
+from slack_sdk.errors import SlackApiError
+from posthog.integrations import IntegrationNotFound
from posthog.temporal.ai_observability.eval_reports.delivery import (
CitationMap,
+ SlackDeliveryFailure,
_build_citation_map,
_format_period_for_display,
_inline_email_styles,
@@ -487,12 +492,35 @@ class TestDeliverReport(SimpleTestCase):
mock_report_qs.select_related.return_value.get.return_value = report
mock_run_qs.get.return_value = run
mock_email.return_value = []
- mock_slack.return_value = ["slack failed"]
+ mock_slack.return_value = [SlackDeliveryFailure(message="slack failed", retryable=True)]
deliver_report("report-id", "run-id")
self.assertEqual(run.delivery_status, "partial_failure")
+ @parameterized.expand([("retryable_raises_for_retry", True), ("config_error_disables_report", False)])
+ @patch("posthog.temporal.ai_observability.eval_reports.delivery.disable_undeliverable_report")
+ @patch("posthog.temporal.ai_observability.eval_reports.delivery.deliver_email_report")
+ @patch("posthog.temporal.ai_observability.eval_reports.delivery.deliver_slack_report")
+ @patch("products.ai_observability.backend.models.evaluation_reports.EvaluationReportRun.objects")
+ @patch("products.ai_observability.backend.models.evaluation_reports.EvaluationReport.objects")
+ def test_all_slack_targets_failed(
+ self, _name, retryable, mock_report_qs, mock_run_qs, mock_slack, _mock_email, mock_disable
+ ):
+ report = self._make_report([{"type": "slack", "integration_id": 1, "channel": "C0123|#reports"}])
+ run = self._make_report_run()
+
+ mock_report_qs.select_related.return_value.get.return_value = report
+ mock_run_qs.get.return_value = run
+ mock_slack.return_value = [SlackDeliveryFailure(message="slack failed", retryable=retryable)]
+
+ with self.assertRaises(RuntimeError) if retryable else nullcontext():
+ deliver_report("report-id", "run-id")
+
+ self.assertEqual(run.delivery_status, "failed")
+ self.assertEqual(run.delivery_errors, ["slack failed"])
+ self.assertEqual(mock_disable.called, not retryable)
+
@patch("posthog.temporal.ai_observability.eval_reports.delivery.deliver_email_report")
@patch("posthog.temporal.ai_observability.eval_reports.delivery.deliver_slack_report")
@patch("products.ai_observability.backend.models.evaluation_reports.EvaluationReportRun.objects")
@@ -678,5 +706,47 @@ class TestDeliverSlackReport(SimpleTestCase):
)
self.assertEqual(len(errors), 1)
- self.assertIn("|#tech-devops-cron", errors[0])
+ self.assertIn("|#tech-devops-cron", errors[0].message)
+ self.assertFalse(errors[0].retryable)
client.chat_postMessage.assert_not_called()
+
+ @parameterized.expand(
+ [
+ ("bot_not_in_channel", True, "not_in_channel", False, "#evals: The PostHog app is not in this channel"),
+ ("revoked_token", True, "token_revoked", False, "Reconnect Slack"),
+ ("rate_limited", True, "ratelimited", True, "ratelimited"),
+ ("integration_deleted", False, None, False, "Reconnect Slack"),
+ ]
+ )
+ @patch("posthog.models.integration.SlackIntegration")
+ @patch("posthog.integrations.IntegrationManager")
+ def test_slack_failure_is_classified(
+ self,
+ _name,
+ integration_exists,
+ slack_error,
+ expected_retryable,
+ expected_text,
+ mock_integration,
+ mock_slack_integration,
+ ):
+ if not integration_exists:
+ mock_integration.slack.oauth.get.side_effect = IntegrationNotFound("deleted")
+ client = MagicMock()
+ client.chat_postMessage.side_effect = SlackApiError("failed", {"ok": False, "error": slack_error})
+ mock_slack_integration.return_value.client = client
+
+ targets = [{"type": "slack", "integration_id": 1, "channel": "C0B5CHB0JQH|#evals"}]
+ errors = deliver_slack_report(
+ self._make_report_run(),
+ targets,
+ evaluation_name="Test Eval",
+ team_id=1,
+ project_id=1,
+ period_start="2026-03-01T00:00:00+00:00",
+ period_end="2026-03-02T00:00:00+00:00",
+ )
+
+ self.assertEqual(len(errors), 1)
+ self.assertEqual(errors[0].retryable, expected_retryable)
+ self.assertIn(expected_text, errors[0].message)