Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(signals): recover scout slack reports slack rejects as too long (#118414)

posthog[bot]0050ff1cc3c3
diff --git a/products/signals/backend/scout_harness/slack_delivery.py b/products/signals/backend/scout_harness/slack_delivery.py
index 088d121f4748..9025a6184759 100644
--- a/products/signals/backend/scout_harness/slack_delivery.py
+++ b/products/signals/backend/scout_harness/slack_delivery.py
@@ -4,7 +4,8 @@ import time
import uuid
from collections.abc import Callable
from dataclasses import dataclass
-from typing import Literal
+from functools import partial
+from typing import Literal, TypeVar
from urllib.parse import quote
from django.conf import settings
@@ -36,6 +37,7 @@ from products.signals.backend.slack_formatting import (
prepare_slack_markdown,
split_markdown_by_headings,
strip_chart_references,
+ truncate_slack_text,
)
from products.signals.backend.slack_report_threads import (
record_report_slack_thread,
@@ -46,6 +48,7 @@ from products.slack_app.backend.facade.api import slack_followup_invite
logger = structlog.get_logger(__name__)
+_MESSAGE_TOO_LONG_ERROR_CODE = "msg_blocks_too_long"
_PERMANENT_SLACK_ERROR_CODES = frozenset(
{
"account_inactive",
@@ -56,6 +59,9 @@ _PERMANENT_SLACK_ERROR_CODES = frozenset(
"is_archived",
"messages_tab_disabled",
"missing_scope",
+ # Permanent only once `_send_shortening_markdown` has resent the message with less text:
+ # a retry rebuilds the same payload from the same report, so it can only fail again.
+ _MESSAGE_TOO_LONG_ERROR_CODE,
"not_authed",
"not_in_channel",
"org_login_required",
@@ -74,8 +80,13 @@ _BLOCK_REJECTION_ERROR_CODES = frozenset({"invalid_blocks", "invalid_blocks_form
_STALE_MESSAGE_ERROR_CODES = frozenset(
{"message_not_found", "cant_update_message", "edit_window_closed", "channel_not_found"}
)
+# Slack does not say how far over its markdown budget a rejected message went, so the resend keeps
+# half of the budget.
+_SHORTENED_MARKDOWN_TEXT_MAX_LEN = SLACK_MARKDOWN_TEXT_MAX_LEN // 2
_SCOUT_SLACK_REPLY_PACE_SECONDS = 1
+_SlackResult = TypeVar("_SlackResult")
+
ScoutSlackOutputType = Literal["finding", "report"]
ScoutSlackReplyRetryScheduler = Callable[[int, list[list[dict]], int, str, str], None]
@@ -295,15 +306,19 @@ def post_scout_emission_to_slack(
client = slack.client
try:
_ensure_dm_recipient_eligible(slack, channel_id)
- client.chat_postMessage(
- channel=channel_id,
- blocks=blocks,
- text=fallback,
- # The queue derives one delivery_id per recipient, so each DM keeps its own
- # Slack idempotency key instead of every recipient sharing the emission id.
- client_msg_id=delivery_id,
- unfurl_links=False,
- unfurl_media=False,
+ _send_shortening_markdown(
+ lambda message_blocks: client.chat_postMessage(
+ channel=channel_id,
+ blocks=message_blocks,
+ text=fallback,
+ # The queue derives one delivery_id per recipient, so each DM keeps its own
+ # Slack idempotency key instead of every recipient sharing the emission id.
+ client_msg_id=delivery_id,
+ unfurl_links=False,
+ unfurl_media=False,
+ ),
+ blocks,
+ delivery_id=delivery_id,
)
except SlackApiError as exc:
error_code = slack_api_error_code(exc)
@@ -315,6 +330,32 @@ def post_scout_emission_to_slack(
raise
+def _shorten_markdown_blocks(blocks: list[dict]) -> list[dict]:
+ return [
+ {**block, "text": truncate_slack_text(block["text"], _SHORTENED_MARKDOWN_TEXT_MAX_LEN)}
+ if block.get("type") == "markdown" and isinstance(block.get("text"), str)
+ else block
+ for block in blocks
+ ]
+
+
+def _send_shortening_markdown(
+ send: Callable[[list[dict]], _SlackResult], blocks: list[dict], *, delivery_id: str
+) -> _SlackResult:
+ """Send the blocks, and send them again with shorter markdown if Slack says they are too long.
+
+ Slack counts its markdown budget in its own way, so text inside our budget can still go over.
+ The report link stays in the message, so a reader still reaches the full text in PostHog."""
+ try:
+ return send(blocks)
+ except SlackApiError as exc:
+ shortened = _shorten_markdown_blocks(blocks)
+ if slack_api_error_code(exc) != _MESSAGE_TOO_LONG_ERROR_CODE or shortened == blocks:
+ raise
+ logger.warning("signals_scout.slack_message_shortened", delivery_id=delivery_id)
+ return send(shortened)
+
+
def _report_header(report: SignalReport) -> str:
title = " ".join((report.title or "").split()) or "New scout report"
return title if len(title) <= 150 else title[:147].rstrip() + "..."
@@ -599,7 +640,10 @@ def _post_scout_report_thread_replies(
# in this worker, but do not send it as one burst that consumes a retry per section.
time.sleep(_SCOUT_SLACK_REPLY_PACE_SECONDS)
try:
- _remember(chunk_index, _post_reply(chunk_index, blocks))
+ _remember(
+ chunk_index,
+ _send_shortening_markdown(partial(_post_reply, chunk_index), blocks, delivery_id=delivery_id),
+ )
except Exception as exc:
error_code = slack_api_error_code(exc) if isinstance(exc, SlackApiError) else None
logger.warning(
@@ -654,7 +698,8 @@ def _post_scout_report_lead_message(
A chart Slack cannot fetch takes the whole message down, so on any deployment Slack cannot reach
a chart-bearing report would never arrive. The prose and the report link are the message; the
- charts are the extra, and the report still draws every one of them in the inbox."""
+ charts are the extra, and the report still draws every one of them in the inbox. A message Slack
+ finds too long is posted again with a shorter summary for the same reason."""
def _post(message_blocks: list[dict]) -> SlackResponse:
if update_ts is not None:
@@ -668,19 +713,22 @@ def _post_scout_report_lead_message(
unfurl_media=False,
)
- try:
- return _post(blocks)
- except SlackApiError as exc:
- error_code = slack_api_error_code(exc)
- if error_code not in _BLOCK_REJECTION_ERROR_CODES or not has_chart_blocks(blocks):
- raise
- logger.warning(
- "signals_scout.slack_report_charts_rejected",
- channel=channel_id,
- delivery_id=delivery_id,
- error_code=error_code,
- )
- return _post(strip_chart_blocks(blocks))
+ def _post_without_rejected_charts(message_blocks: list[dict]) -> SlackResponse:
+ try:
+ return _post(message_blocks)
+ except SlackApiError as exc:
+ error_code = slack_api_error_code(exc)
+ if error_code not in _BLOCK_REJECTION_ERROR_CODES or not has_chart_blocks(message_blocks):
+ raise
+ logger.warning(
+ "signals_scout.slack_report_charts_rejected",
+ channel=channel_id,
+ delivery_id=delivery_id,
+ error_code=error_code,
+ )
+ return _post(strip_chart_blocks(message_blocks))
+
+ return _send_shortening_markdown(_post_without_rejected_charts, blocks, delivery_id=delivery_id)
def post_scout_report_to_slack(
diff --git a/products/signals/backend/slack_formatting.py b/products/signals/backend/slack_formatting.py
index 313ca54a88a1..f01d08d72426 100644
--- a/products/signals/backend/slack_formatting.py
+++ b/products/signals/backend/slack_formatting.py
@@ -100,11 +100,38 @@ def is_safe_slack_http_url(value: object) -> bool:
return not any(char in value for char in ("<", ">", "|"))
+def _slack_char_length(char: str) -> int:
+ if char == "&":
+ return 5
+ if char in "<>":
+ return 4
+ return 2 if ord(char) > 0xFFFF else 1
+
+
+def slack_text_length(text: str) -> int:
+ """The length Slack can count for `text` against a block's character budget.
+
+ Slack can measure the text after it escapes `&`, `<`, and `>`, and in UTF-16 code units, where a
+ character outside the Basic Multilingual Plane costs two. The worst case of both keeps a message
+ near the cap from going over."""
+ return sum(_slack_char_length(char) for char in text)
+
+
+def _slack_fit_index(text: str, limit: int) -> int:
+ """The longest prefix of `text`, as an index, whose `slack_text_length` stays within `limit`."""
+ used = 0
+ for index, char in enumerate(text):
+ used += _slack_char_length(char)
+ if used > limit:
+ return index
+ return len(text)
+
+
def truncate_slack_text(text: str, limit: int) -> str:
"""Keep text below the block's character limit with headroom."""
- if len(text) <= limit:
+ if slack_text_length(text) <= limit:
return text
- return text[: limit - 1].rstrip() + "…"
+ return text[: _slack_fit_index(text, limit - 1)].rstrip() + "…"
def prepare_slack_markdown(text: str) -> str:
@@ -307,25 +334,30 @@ def chunk_slack_text(text: str, limit: int) -> list[str]:
text = text.strip()
if not text:
return []
- if len(text) <= limit:
+ if slack_text_length(text) <= limit:
return [text]
chunks: list[str] = []
current = ""
+ current_length = 0
for line in text.split("\n"):
- while len(line) > limit:
+ while slack_text_length(line) > limit:
if current:
chunks.append(current.rstrip())
current = ""
- cut = _line_break_index(line, limit)
+ current_length = 0
+ cut = _line_break_index(line, _slack_fit_index(line, limit))
chunks.append(line[:cut].rstrip())
line = line[cut:]
- candidate = f"{current}\n{line}" if current else line
- if len(candidate) > limit:
+ line_length = slack_text_length(line)
+ candidate_length = current_length + 1 + line_length if current else line_length
+ if candidate_length > limit:
if current:
chunks.append(current.rstrip())
current = line
+ current_length = line_length
else:
- current = candidate
+ current = f"{current}\n{line}" if current else line
+ current_length = candidate_length
if current.strip():
chunks.append(current.rstrip())
return chunks
diff --git a/products/signals/backend/test/test_scout_slack_delivery.py b/products/signals/backend/test/test_scout_slack_delivery.py
index 815e56f22bc8..a2eff8608014 100644
--- a/products/signals/backend/test/test_scout_slack_delivery.py
+++ b/products/signals/backend/test/test_scout_slack_delivery.py
@@ -1393,6 +1393,54 @@ class TestScoutSlackDelivery(BaseTest):
]
assert [b["type"] for b in resent] == ["context", "header", "markdown", "actions", "context"]
+ @parameterized.expand([("shorter_summary_fits", 1, True), ("shorter_summary_still_too_long", 2, False)])
+ def test_report_too_long_for_slack_is_resent_with_a_shorter_summary(
+ self, _name: str, rejections: int, expect_delivered: bool
+ ) -> None:
+ emission = self._make_emission()
+ report = SignalReport.objects.create(
+ team=self.team,
+ status=SignalReport.Status.READY,
+ title="Checkout failures",
+ summary="Checkout & payments failed. " * 500,
+ )
+ integration = Integration.objects.create(team=self.team, kind=Integration.IntegrationKind.SLACK)
+ fake_client = MagicMock()
+ too_long = SlackApiError(message="too long", response=FakeSlackResponse({"error": "msg_blocks_too_long"}))
+ fake_client.chat_postMessage.side_effect = [too_long] * rejections + [
+ FakeSlackResponse({"ts": "1785418710.000700"})
+ ]
+
+ with (
+ patch("products.signals.backend.scout_harness.slack_delivery.SlackIntegration") as slack_integration,
+ patch("products.signals.backend.tasks.capture_exception") as capture,
+ ):
+ slack_integration.return_value.client = fake_client
+ deliver_scout_slack_output.apply(
+ args=(
+ self.team.id,
+ "report",
+ str(report.id),
+ str(emission.scout_run_id),
+ "01864f4c-6957-7d3f-8d85-1d775e527266",
+ integration.id,
+ "CSCOUTS|#scout-findings",
+ ),
+ throw=True,
+ )
+
+ posts = [call.kwargs["blocks"] for call in fake_client.chat_postMessage.call_args_list]
+ assert len(posts) == 2
+ first_markdown, resent_markdown = (
+ next(block["text"] for block in blocks if block["type"] == "markdown") for blocks in posts
+ )
+ assert len(resent_markdown) < len(first_markdown)
+ assert any(block["type"] == "actions" for block in posts[1])
+ if expect_delivered:
+ capture.assert_not_called()
+ else:
+ assert capture.call_args.args[1]["error_code"] == "msg_blocks_too_long"
+
def test_task_retries_transient_delivery_failure(self) -> None:
emission = self._make_emission()
error = SlackApiError(
diff --git a/products/signals/backend/test/test_slack_formatting.py b/products/signals/backend/test/test_slack_formatting.py
index e0300a63e815..092c63d24f57 100644
--- a/products/signals/backend/test/test_slack_formatting.py
+++ b/products/signals/backend/test/test_slack_formatting.py
@@ -11,6 +11,7 @@ from products.signals.backend.slack_formatting import (
chunk_slack_text,
defuse_slack_tokens,
group_segments_to_limit,
+ slack_text_length,
split_markdown_by_headings,
strip_chart_references,
)
@@ -295,15 +296,17 @@ class TestChunkSlackText(SimpleTestCase):
[
("plain_run", "x" * (SLACK_MARKDOWN_TEXT_MAX_LEN * 2 + 5)),
("link_longer_than_a_chunk", "https://example.com/" + "z" * (SLACK_MARKDOWN_TEXT_MAX_LEN * 2)),
+ ("characters_slack_escapes", "a&b<c>" * (SLACK_MARKDOWN_TEXT_MAX_LEN // 4)),
]
)
def test_unbreakable_run_is_hard_sliced(self, _name: str, line: str) -> None:
# A run with no sentence, word, or token boundary has nowhere safe to break, so it is sliced
# at the limit. Guards both halves of that fallback: it has to terminate, and it has to keep
- # every character rather than dropping the remainder.
+ # every character rather than dropping the remainder. Slack can count `&`, `<` and `>` after
+ # it escapes them, so the limit holds for the escaped length.
chunks = chunk_slack_text(line, SLACK_MARKDOWN_TEXT_MAX_LEN)
- assert all(len(chunk) <= SLACK_MARKDOWN_TEXT_MAX_LEN for chunk in chunks)
+ assert all(slack_text_length(chunk) <= SLACK_MARKDOWN_TEXT_MAX_LEN for chunk in chunks)
assert "".join(chunks) == line
@parameterized.expand(