:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import json
from datetime import datetime, timedelta
from types import SimpleNamespace
from typing import Any
import pytest
import time_machine
from unittest.mock import MagicMock, patch
from django.db import OperationalError
from django.db.models import QuerySet
from django.test import override_settings
from django.utils import timezone
from owners_yaml.schema import TeamEntry
from parameterized import parameterized
from slack_sdk.errors import SlackApiError
from structlog.testing import capture_logs
from posthog.models.integration import Integration
from posthog.models.scoping import team_scope
from posthog.slack.channels import SlackChannel
from products.stamphog.backend.facade.enums import AudienceReason, ChannelResolutionSource, DigestRunStatus
from products.stamphog.backend.logic.audiences import REPO_AUDIENCE_PREFIX
from products.stamphog.backend.logic.channel_resolution import (
Destination,
RoutingContext,
RoutingUnavailable,
_candidate_repo_configs,
)
from products.stamphog.backend.logic.digest import (
_HEADLINE_MAX_RETRIES,
_HEADLINE_MAX_TOKENS,
_HEADLINE_TIMEOUT_SECONDS,
GRAZE_CHANGED_FILES,
MAX_DIGEST_PRS,
MAX_FALLBACK_PRS,
SCOPE_YOUR_FILES,
DigestPRSummary,
DigestSummary,
_audience_team_slug,
_build_headline_prompt,
_build_selection_prompt,
_build_summary,
_fallback_summary,
_parse_selection,
summarize_merged_prs,
)
from products.stamphog.backend.logic.digest_runs import (
DIGEST_LOOKBACK_DAYS,
STALE_PENDING_RUN_MINUTES,
_previous_run_slot,
pending_audiences_by_team,
reclaim_stale_pending_runs,
)
from products.stamphog.backend.logic.slack_digest import DigestSlackError, post_digest_details, post_digest_lead
from products.stamphog.backend.models import DigestRun, PullRequest, PullRequestAudience, StamphogRepoConfig
from products.stamphog.backend.tasks.digest import send_team_digests
from products.stamphog.backend.tests.conftest import PRODUCT_DATABASES
from products.stamphog.backend.tests.fakes import FakeSlackIntegration
REPO = "acme/widgets"
AUDIENCE = "team-devex"
def _summary(prs: list[PullRequest], audiences: list | None = None) -> DigestSummary:
"""Stand in for the LLM so the task never reaches a gateway. Keeps every PR: a summary that
keeps nothing is its own path (the digest posts nothing and consumes the claim).
Goes through _build_summary rather than building a DigestSummary, so a task test sees the same
rail a real run would."""
return _build_summary(
len(prs),
[
DigestPRSummary(
pr_number=pr.pr_number,
title=pr.title,
url=pr.pr_url,
author_login=pr.author_login,
summary=pr.title,
repository=pr.repo_config.repository,
)
for pr in prs
],
)
def _seed_prs(
team_id: int,
pr_count: int = 2,
repository: str = REPO,
first_number: int = 1,
digest_enabled: bool = True,
) -> StamphogRepoConfig:
repo_config = StamphogRepoConfig.objects.for_team(team_id).create(
team_id=team_id, repository=repository, installation_id="9001", digest_enabled=digest_enabled
)
for number in range(first_number, first_number + pr_count):
pr = PullRequest.objects.for_team(team_id).create(
team_id=team_id,
repo_config=repo_config,
pr_number=number,
title=f"Change number {number}",
author_login="devex-dev",
pr_url=f"https://github.com/{repository}/pull/{number}",
merged_at=timezone.now(),
)
PullRequestAudience.objects.for_team(team_id).create(
team_id=team_id, pull_request=pr, audience_key=AUDIENCE, reason=AudienceReason.OWNED
)
return repo_config
def _channel(channel_id: str, shared: bool = False) -> SlackChannel:
return SlackChannel(channel_id=channel_id, shared=shared)
def _routing_context(
registry_by_repo: dict[str, dict[str, TeamEntry]] | None = None,
declared_repo_channel: dict[str, str] | None = None,
channels_by_name: dict[str, SlackChannel] | None = None,
) -> RoutingContext:
"""A routing context built in memory, so a task test never reaches GitHub or Slack.
The default routes AUDIENCE to C1 through the plain name match, which is what most of these
tests want: they are about claiming and posting, not about how the destination was decided.
"""
return RoutingContext(
slack_integration_id=1,
registry_by_repo=registry_by_repo if registry_by_repo is not None else {REPO: {}},
inherited_registry=next((r for r in (registry_by_repo or {}).values() if r), {}),
declared_repo_channel=declared_repo_channel or {},
channels_by_name=channels_by_name if channels_by_name is not None else {AUDIENCE: _channel("C1")},
)
def _run_digests(team_id: int, context: RoutingContext | None = None) -> None:
with patch(
"products.stamphog.backend.logic.digest_runs.build_routing_context",
return_value=_routing_context() if context is None else context,
):
send_team_digests(team_id=team_id, audience_keys=[AUDIENCE])
@pytest.mark.parametrize(
"slack_ts,expect_status,expect_prs_linked",
[("1234.5", DigestRunStatus.COMPLETED, True), ("", DigestRunStatus.FAILED, False)],
ids=["posted_run_finalized_keeps_prs", "unposted_run_reclaimed_unlinks_prs"],
)
@pytest.mark.django_db(databases=PRODUCT_DATABASES)
def testreclaim_stale_pending_runs(team, slack_ts, expect_status, expect_prs_linked) -> None:
# A worker that dies mid-run leaves a PENDING run with its PRs claimed. If it already posted to Slack
# (slack_message_ts set), reclaim must finalize it as COMPLETED and KEEP its PRs linked so the next
# digest doesn't re-send them. If it never posted, reclaim unlinks the PRs so they're retried.
with team_scope(team.id):
_seed_prs(team.id, pr_count=2)
run = DigestRun.objects.for_team(team.id).create(
team_id=team.id,
audience_key=AUDIENCE,
slack_channel_id="C1",
status=DigestRunStatus.PENDING,
slack_message_ts=slack_ts,
)
PullRequestAudience.objects.for_team(team.id).filter(audience_key=AUDIENCE).update(digest_run=run)
stale = timezone.now() - timedelta(minutes=STALE_PENDING_RUN_MINUTES + 5)
DigestRun.objects.for_team(team.id).filter(id=run.id).update(created_at=stale)
reclaim_stale_pending_runs()
with team_scope(team.id):
run.refresh_from_db()
linked = PullRequestAudience.objects.for_team(team.id).filter(digest_run_id=run.id).count()
assert run.status == expect_status
assert (linked == 2) is expect_prs_linked
@pytest.mark.django_db(databases=PRODUCT_DATABASES)
def test_proof_of_post_persists_metadata_for_reclaim(team) -> None:
# Worker death between Slack accepting the message and the completion write: the reclaim sweeper
# finalizes from persisted state only, so the proof-of-post write must already carry
# pr_count/summary, or the finalized run keeps zeros while its PRs stay linked.
#
# The crash goes in at the thread post, which is the last call before the completion write.
# An earlier version raised on the second transaction.atomic call; the completion write is no
# longer wrapped in one, so that injection stopped firing and the test passed on the ordinary
# path without ever entering the window it names.
with team_scope(team.id):
_seed_prs(team.id, pr_count=2)
with (
patch("products.stamphog.backend.logic.digest_runs.summarize_merged_prs", side_effect=_summary),
patch("products.stamphog.backend.logic.digest_runs.post_digest_lead", return_value="1234.5"),
patch(
"products.stamphog.backend.logic.digest_runs.post_digest_details",
side_effect=RuntimeError("worker died before the completion write"),
),
):
# send_team_digests contains each audience's failure, so the crash is read off the run state
# rather than raised out of the task.
_run_digests(team.id)
with team_scope(team.id):
stranded = DigestRun.objects.for_team(team.id).get()
# The window this test exists for: Slack has the message, the run does not say so yet.
assert stranded.status == DigestRunStatus.PENDING
assert stranded.slack_message_ts == "1234.5"
DigestRun.objects.for_team(team.id).update(
created_at=timezone.now() - timedelta(minutes=STALE_PENDING_RUN_MINUTES + 5)
)
reclaim_stale_pending_runs()
with team_scope(team.id):
run = DigestRun.objects.for_team(team.id).get()
assert run.status == DigestRunStatus.COMPLETED
assert run.pr_count == 2
assert run.summary # the summary rode along with the proof-of-post, not just the message ts
@pytest.mark.parametrize(
"fail_times,expect_pending",
[(2, False), (3, True)],
ids=["retries_then_succeeds", "exhausts_and_leaves_the_run_for_the_sweeper"],
)
@pytest.mark.django_db(databases=PRODUCT_DATABASES)
def test_proof_of_post_write_retries_transient_db_error(team, fail_times: int, expect_pending: bool) -> None:
# The proof-of-post write is the dedup proof: once Slack accepts, only slack_message_ts stops the
# reclaim sweeper from re-sending. A transient DB blip there must be retried (not taken at face
# value) or it converts into a duplicate Slack post. Slack is posted exactly once regardless; if the
# write never lands, the run stays PENDING with its PRs linked — the crash-window semantics the
# reclaim sweeper then handles.
_seed_prs(team.id, pr_count=2)
attempts = {"n": 0}
real_update = QuerySet.update
def flaky_update(self: Any, **kwargs: Any) -> int:
# Target only the proof-of-post write: it sets slack_message_ts but, unlike the completion
# write, carries no status/posted_at.
is_proof = "slack_message_ts" in kwargs and "status" not in kwargs and "posted_at" not in kwargs
if is_proof:
attempts["n"] += 1
if attempts["n"] <= fail_times:
raise OperationalError("transient db blip")
return real_update(self, **kwargs)
post = MagicMock(return_value="1234.5")
sleeps: list[float] = []
with (
patch("products.stamphog.backend.logic.digest_runs.post_digest_lead", post),
patch("products.stamphog.backend.logic.digest_runs.summarize_merged_prs", side_effect=_summary),
patch("products.stamphog.backend.logic.digest_runs.time.sleep", side_effect=lambda s: sleeps.append(s)),
patch.object(QuerySet, "update", flaky_update),
):
_run_digests(team.id)
assert post.call_count == 1 # Slack posted exactly once either way
with team_scope(team.id):
run = DigestRun.objects.get()
linked = PullRequestAudience.objects.filter(digest_run_id=run.id).count()
if expect_pending:
assert run.status == DigestRunStatus.PENDING # never finalized
assert linked == 2 # PRs stay linked to the PENDING run for the reclaim sweeper
assert len(sleeps) == fail_times - 1 # slept between the 3 attempts, not after the last
else:
assert run.status == DigestRunStatus.COMPLETED
assert run.pr_count == 2
assert len(sleeps) == fail_times
# ---- Finding 2: an audience's digest must never post to Slack twice ----------------------------
@pytest.mark.django_db(databases=PRODUCT_DATABASES)
def test_concurrent_runs_for_one_audience_post_to_slack_once(team) -> None:
# Two workers firing for the same audience would both read the same unlinked PRs and both post.
# The fix claims the PRs (links them to a run) before posting, so a second worker that starts
# mid-post finds nothing unlinked and returns without posting. Re-entering post_digest simulates
# that overlap deterministically.
_seed_prs(team.id)
posts: list[str] = []
def reentrant_post(team_id: int, destination: Any, summary: Any) -> str:
posts.append(destination.channel_id)
if len(posts) == 1: # a second worker starts while the first is posting
_run_digests(team_id)
return f"ts-{len(posts)}"
with (
patch("products.stamphog.backend.logic.digest_runs.post_digest_lead", side_effect=reentrant_post),
patch("products.stamphog.backend.logic.digest_runs.summarize_merged_prs", side_effect=_summary),
):
_run_digests(team.id)
assert len(posts) == 1 # the re-entrant worker found no unlinked PRs and did not post
with team_scope(team.id):
completed = list(DigestRun.objects.filter(status=DigestRunStatus.COMPLETED))
assert len(completed) == 1
assert PullRequestAudience.objects.filter(digest_run__isnull=True).count() == 0
@pytest.mark.django_db(databases=PRODUCT_DATABASES)
def test_two_repos_declaring_one_team_partition_rather_than_duplicate(team) -> None:
# A repo declaring a team's channel is a scope, not a conflict: it answers for its own merges
# and leaves everyone else's alone. Picking a winner discards a declaration somebody wrote, and
# fanning out posts every merge twice in the product whose problem is volume. Each PR must
# appear in exactly one digest.
_seed_prs(team.id, pr_count=2, repository=REPO, first_number=1)
_seed_prs(team.id, pr_count=3, repository="acme/gadgets", first_number=10)
context = _routing_context(
registry_by_repo={
# REPO carries a registry that does not mention this team, which is an answer: the
# derived name is right. Without that, gadgets' declaration would capture REPO's merges
# too, since REPO declared nothing to the contrary.
REPO: {"team-other": TeamEntry(slack="#team-other")},
"acme/gadgets": {AUDIENCE: TeamEntry(notifications="#bots-devex")},
},
channels_by_name={AUDIENCE: _channel("C1"), "bots-devex": _channel("C2")},
)
posted: dict[str, int] = {}
def record(team_id: int, destination: Any, summary: Any) -> str:
posted[destination.channel_id] = len(summary.prs)
return f"ts-{destination.channel_id}"
with (
patch("products.stamphog.backend.logic.digest_runs.post_digest_lead", side_effect=record),
patch("products.stamphog.backend.logic.digest_runs.summarize_merged_prs", side_effect=_summary),
):
_run_digests(team.id, context)
assert posted == {"C1": 2, "C2": 3}
with team_scope(team.id):
assert PullRequestAudience.objects.filter(digest_run__isnull=True).count() == 0
sources = set(DigestRun.objects.values_list("resolution_source", flat=True))
assert sources == {ChannelResolutionSource.SLACK_NAME_MATCH, ChannelResolutionSource.OWNERS_CONTACT}
@pytest.mark.django_db(databases=PRODUCT_DATABASES)
def test_switching_a_repos_digest_off_stops_merges_it_already_captured(team) -> None:
# The toggle used to be read only at capture, which decides what gets stamped. A repo switched
# off after its merges landed therefore kept posting them for the rest of the claim window, and
# the owner who switched it off had no way to stop that. Reading it at claim time is what makes
# the toggle take effect on the next digest rather than on the next merge.
repo_config = _seed_prs(team.id, pr_count=3)
with team_scope(team.id):
StamphogRepoConfig.objects.filter(id=repo_config.id).update(digest_enabled=False)
assert pending_audiences_by_team().get(team.id) is None
with (
patch("products.stamphog.backend.logic.digest_runs.post_digest_lead", return_value="ts-1") as post,
patch("products.stamphog.backend.logic.digest_runs.summarize_merged_prs", side_effect=_summary),
):
_run_digests(team.id)
assert not post.called
with team_scope(team.id):
assert PullRequestAudience.objects.filter(digest_run__isnull=True).count() == 3
# Switching it back on returns the backlog still inside the claim floor, rather than losing it.
with team_scope(team.id):
StamphogRepoConfig.objects.filter(id=repo_config.id).update(digest_enabled=True)
with (
patch("products.stamphog.backend.logic.digest_runs.post_digest_lead", return_value="ts-1") as post,
patch("products.stamphog.backend.logic.digest_runs.summarize_merged_prs", side_effect=_summary),
):
_run_digests(team.id)
assert post.called
with team_scope(team.id):
assert PullRequestAudience.objects.filter(digest_run__isnull=True).count() == 0
@pytest.mark.django_db(databases=PRODUCT_DATABASES)
def test_unroutable_merges_do_not_starve_routable_ones_behind_them(team) -> None:
# The cap used to be applied before routing. An audience whose oldest merges all came from an
# unroutable repo filled the whole claim with rows the run then dropped, and because dropped
# rows stay unclaimed the next run selected exactly the same ones. The routable merges behind
# them were never reached and aged out of the window unposted.
_seed_prs(team.id, pr_count=2, repository="acme/nowhere", first_number=1)
_seed_prs(team.id, pr_count=2, repository=REPO, first_number=10)
context = _routing_context(
registry_by_repo={REPO: {}}, # acme/nowhere is not a candidate repo, so it routes nowhere
channels_by_name={AUDIENCE: _channel("C1")},
)
with (
patch("products.stamphog.backend.logic.digest_runs.post_digest_lead", return_value="ts-1") as post,
patch("products.stamphog.backend.logic.digest_runs.summarize_merged_prs", side_effect=_summary),
patch("products.stamphog.backend.logic.digest_runs.DIGEST_MAX_PRS_PER_RUN", 2),
):
_run_digests(team.id, context)
assert post.called
_team_id, _destination, posted = post.call_args.args
assert {pr.pr_number for pr in posted.prs} == {10, 11}
@pytest.mark.django_db(databases=PRODUCT_DATABASES)
def test_a_repo_with_no_registry_inherits_one(team) -> None:
# The convenience layer. charts carries no ownership file at all, so a team whose channel is not
# named after its slug still routes correctly there because the monorepo says where it goes.
_seed_prs(team.id, pr_count=2, repository="acme/charts")
context = _routing_context(
registry_by_repo={"acme/charts": {}, REPO: {AUDIENCE: TeamEntry(slack="#team-apm")}},
channels_by_name={"team-apm": _channel("C9")},
)
with (
patch("products.stamphog.backend.logic.digest_runs.post_digest_lead", return_value="ts-1") as post,
patch("products.stamphog.backend.logic.digest_runs.summarize_merged_prs", side_effect=_summary),
):
_run_digests(team.id, context)
_team_id, destination, _posted = post.call_args.args
assert destination.channel_id == "C9"
@pytest.mark.parametrize(
"context_kwargs,logged",
[
(
{"registry_by_repo": {REPO: {AUDIENCE: TeamEntry(notifications=False)}}},
("stamphog_digest_opted_out", "info"),
),
# A team that silences this digest alone keeps its channel for every other bot.
(
{"registry_by_repo": {REPO: {AUDIENCE: TeamEntry(slack="#team-apm", notifications={"stamphog": False})}}},
("stamphog_digest_opted_out", "info"),
),
({"channels_by_name": {}}, ("stamphog_digest_no_destination", "warning")),
],
ids=["silenced_everywhere", "silenced_for_this_digest_only", "no_channel_of_that_name"],
)
@pytest.mark.django_db(databases=PRODUCT_DATABASES)
def test_unroutable_merges_stay_unclaimed(team, context_kwargs: dict, logged: tuple[str, str]) -> None:
# Claiming marks a PR as handled forever, so a merge that routes nowhere must not be claimed.
# A team that silences its digest today and declares a channel next week has to receive the
# merges in between, and a channel created after the declaration has to pick up the backlog.
_seed_prs(team.id, pr_count=2)
with (
patch("products.stamphog.backend.logic.digest_runs.post_digest_lead") as post,
patch("products.stamphog.backend.logic.digest_runs.summarize_merged_prs", side_effect=_summary),
capture_logs() as logs,
):
_run_digests(team.id, _routing_context(**context_kwargs))
assert not post.called
routing_events = [
(entry["event"], entry["log_level"])
for entry in logs
if entry["event"] in {"stamphog_digest_opted_out", "stamphog_digest_no_destination"}
]
assert routing_events == [logged]
with team_scope(team.id):
assert PullRequestAudience.objects.filter(digest_run__isnull=True).count() == 2
assert not DigestRun.objects.exists()
@pytest.mark.django_db(databases=PRODUCT_DATABASES)
def test_an_unreadable_registry_posts_nothing(team) -> None:
# Routing is derived every run and never cached, so a half-read registry does not degrade — it
# silently reroutes. The repo whose fetch failed could be the one declaring every team's
# channel, and continuing without it would send a whole morning of digests to derived names.
_seed_prs(team.id, pr_count=2)
with (
patch(
"products.stamphog.backend.logic.digest_runs.build_routing_context",
side_effect=RoutingUnavailable("github is down"),
),
patch("products.stamphog.backend.logic.digest_runs.post_digest_lead") as post,
):
send_team_digests(team_id=team.id, audience_keys=[AUDIENCE])
assert not post.called
with team_scope(team.id):
assert PullRequestAudience.objects.filter(digest_run__isnull=True).count() == 2
@pytest.mark.django_db(databases=PRODUCT_DATABASES)
def test_a_blank_installation_placeholder_is_not_a_routing_candidate(team) -> None:
# A repo created through the API carries a blank installation and defaults to enabled, so it
# used to join the candidates. It can fetch no routing file, and every candidate is read, so the
# failed fetch raised RoutingUnavailable and silenced the whole team's digest. This does not
# relax that rule: a repo that was readable and broke still stops the run. A blank installation
# was never readable, resolves no webhook, and so reports no merges either.
repo_config = _seed_prs(team.id, pr_count=1)
with team_scope(team.id):
placeholder = StamphogRepoConfig.objects.for_team(team.id).create(
team_id=team.id, repository="acme/placeholder", installation_id="", enabled=True
)
candidates = _candidate_repo_configs(team.id)
assert [config.id for config in candidates] == [repo_config.id]
assert placeholder.id not in {config.id for config in candidates}
@pytest.mark.django_db(databases=PRODUCT_DATABASES)
def test_claim_is_capped_per_run_and_backlog_drains_across_runs(team) -> None:
# An unbounded claim grows the LLM prompt and the Slack payload with the merge-burst size, and a
# rejected oversized payload retries the identical batch forever. The claim caps per run and the
# remainder drains on the next one.
_seed_prs(team.id, pr_count=3)
batch_sizes: list[int] = []
def sized_summary(prs: list[PullRequest], audiences: list | None = None) -> DigestSummary:
batch_sizes.append(len(prs))
return _summary(prs)
with (
patch("products.stamphog.backend.logic.digest_runs.DIGEST_MAX_PRS_PER_RUN", 2),
patch("products.stamphog.backend.logic.digest_runs.post_digest_lead", return_value="ts-1"),
patch("products.stamphog.backend.logic.digest_runs.summarize_merged_prs", side_effect=sized_summary),
):
_run_digests(team.id)
_run_digests(team.id)
assert batch_sizes == [2, 1]
with team_scope(team.id):
assert PullRequestAudience.objects.filter(digest_run__isnull=True).count() == 0
@pytest.mark.django_db(databases=PRODUCT_DATABASES)
def test_prs_the_rail_left_out_stay_consumed(team) -> None:
# The rail is a Slack payload limit and never an editorial one, so what it removes is dropped
# rather than handed back. Releasing the overflow put the same merges in front of the same
# prompt every morning, which grew a team's claim instead of draining it and carried merges
# into a channel days after they landed.
overflow = 2
_seed_prs(team.id, pr_count=MAX_DIGEST_PRS + overflow)
with (
patch("products.stamphog.backend.logic.digest_runs.post_digest_lead", return_value="ts-1") as post,
patch("products.stamphog.backend.logic.digest_runs.summarize_merged_prs", side_effect=_summary),
):
_run_digests(team.id)
assert post.called
_team_id, _destination, posted = post.call_args.args
assert len(posted.prs) == MAX_DIGEST_PRS
with team_scope(team.id):
assert PullRequestAudience.objects.filter(digest_run__isnull=True).count() == 0
assert PullRequestAudience.objects.filter(digest_run__isnull=False).count() == MAX_DIGEST_PRS + overflow
@pytest.mark.django_db(databases=PRODUCT_DATABASES)
def test_a_summary_that_keeps_nothing_consumes_the_claim(team) -> None:
# Keeping nothing is a judgment, not a failure worth retrying. Releasing the claim sent the same
# merges back to the same prompt the next morning, where the same text produced the same answer,
# so the batch grew for a week and then aged out unseen.
_seed_prs(team.id, pr_count=3)
with (
patch("products.stamphog.backend.logic.digest_runs.post_digest_lead") as post,
patch(
"products.stamphog.backend.logic.digest_runs.summarize_merged_prs",
return_value=_build_summary(3, []),
),
):
_run_digests(team.id)
assert not post.called
with team_scope(team.id):
assert PullRequestAudience.objects.filter(digest_run__isnull=True).count() == 0
run = DigestRun.objects.for_team(team.id).get()
assert run.status == DigestRunStatus.COMPLETED
assert run.pr_count == 3
@pytest.mark.django_db(databases=PRODUCT_DATABASES)
def test_failed_slack_post_leaves_prs_retryable_next_run(team) -> None:
# A Slack failure must not hide the PRs: they're claimed before posting, so on failure they have
# to be unlinked again (the retry query filters digest_run__isnull=True). Otherwise they'd stay
# bound to a FAILED run and never retry.
_seed_prs(team.id)
with (
patch("products.stamphog.backend.logic.digest_runs.post_digest_lead", side_effect=RuntimeError("slack down")),
patch("products.stamphog.backend.logic.digest_runs.summarize_merged_prs", side_effect=_summary),
):
_run_digests(team.id)
with team_scope(team.id):
run = DigestRun.objects.get()
assert run.status == DigestRunStatus.FAILED
assert PullRequestAudience.objects.filter(digest_run__isnull=True).count() == 2 # unlinked, retryable
with (
patch("products.stamphog.backend.logic.digest_runs.post_digest_lead", return_value="ts-ok"),
patch("products.stamphog.backend.logic.digest_runs.summarize_merged_prs", side_effect=_summary),
):
_run_digests(team.id)
with team_scope(team.id):
completed = DigestRun.objects.get(status=DigestRunStatus.COMPLETED)
assert PullRequestAudience.objects.filter(digest_run=completed).count() == 2 # retry picked them up
@pytest.mark.parametrize(
"now,expected",
[
# Wednesday 08:00 -> previous slot is Tuesday 07:00
("2026-07-15T08:00:00+00:00", "2026-07-14T07:00:00+00:00"),
# Monday 08:00 -> previous slot is Friday 07:00 (weekend has no slot)
("2026-07-13T08:00:00+00:00", "2026-07-10T07:00:00+00:00"),
# before today's slot -> current slot is yesterday's, previous the day before
("2026-07-15T06:00:00+00:00", "2026-07-13T07:00:00+00:00"),
],
ids=["midweek", "monday_covers_weekend", "before_todays_slot"],
)
def test_previous_run_slot(now: str, expected: str) -> None:
assert _previous_run_slot(datetime.fromisoformat(now)) == datetime.fromisoformat(expected)
@pytest.mark.parametrize(
"has_history,claimed_offset,unclaimed_offset",
[
# first digest: only the previous cadence slot onward — a day-old backlog PR is out
(False, timedelta(hours=19), timedelta(hours=43)),
# established channel: wide floor for failed-run resilience, but a week+ old PR is out
(True, timedelta(hours=43), timedelta(days=DIGEST_LOOKBACK_DAYS + 1)),
],
ids=["first_digest_cadence_window", "established_audience_week_floor"],
)
@pytest.mark.django_db(databases=PRODUCT_DATABASES)
@time_machine.travel("2026-07-15T08:00:00+00:00", tick=False) # a Wednesday; previous slot = Tue 07:00 UTC
def test_digest_claim_floor(team, has_history: bool, claimed_offset: timedelta, unclaimed_offset: timedelta) -> None:
# An audience's first digest must cover only the natural cadence window (what it would have
# received had it been routable one run earlier), never an arbitrary backlog; an established
# audience keeps the wide week floor so merges from a failed run are retried instead of aging
# out fast.
with team_scope(team.id):
repo_config = StamphogRepoConfig.objects.for_team(team.id).create(
team_id=team.id, repository=REPO, installation_id="9001", digest_enabled=True
)
if has_history:
DigestRun.objects.for_team(team.id).create(
team_id=team.id,
audience_key=AUDIENCE,
slack_channel_id="C1",
status=DigestRunStatus.COMPLETED,
)
recent = PullRequestAudience.objects.for_team(team.id).create(
team_id=team.id,
pull_request=PullRequest.objects.for_team(team.id).create(
team_id=team.id,
repo_config=repo_config,
pr_number=1,
merged_at=timezone.now() - claimed_offset,
),
audience_key=AUDIENCE,
reason=AudienceReason.OWNED,
)
old = PullRequestAudience.objects.for_team(team.id).create(
team_id=team.id,
pull_request=PullRequest.objects.for_team(team.id).create(
team_id=team.id,
repo_config=repo_config,
pr_number=2,
merged_at=timezone.now() - unclaimed_offset,
),
audience_key=AUDIENCE,
reason=AudienceReason.OWNED,
)
with (
patch("products.stamphog.backend.logic.digest_runs.post_digest_lead", return_value="ts-ok"),
patch("products.stamphog.backend.logic.digest_runs.summarize_merged_prs", side_effect=_summary),
):
_run_digests(team.id)
with team_scope(team.id):
recent.refresh_from_db()
old.refresh_from_db()
assert recent.digest_run_id is not None # within window -> claimed and digested
assert old.digest_run_id is None # outside window -> left for no one, never flooded in
# ---- Finding 3: same PR number from different repos must not collapse --------------------------
def _pr_stub(repository: str, pr_number: int, title: str, url: str) -> PullRequest:
"""Unsaved PullRequest with just the fields the summarizer reads — no DB needed."""
repo_config = StamphogRepoConfig(repository=repository, installation_id="x")
return PullRequest(
repo_config=repo_config,
team_id=7,
pr_number=pr_number,
title=title,
pr_url=url,
author_login="dev",
additions=1,
deletions=0,
changed_files=1,
body_excerpt="",
)
def _recording_llm_client(answers: list[Any]) -> Any:
"""Answers each call with the next entry, recording the prompts it was given on `.prompts`.
An entry that is an exception is raised instead, which is how the selection call and the
headline call are given different fates.
"""
prompts: list[str] = []
calls: list[dict[str, Any]] = []
def create(**kwargs: Any) -> Any:
calls.append(kwargs)
prompts.append(kwargs["messages"][0]["content"])
answer = answers[len(prompts) - 1]
if isinstance(answer, Exception):
raise answer
# The Anthropic Messages shape: a list of content blocks, text ones carry the answer.
return SimpleNamespace(content=[SimpleNamespace(type="text", text=answer)])
client = SimpleNamespace(messages=SimpleNamespace(create=create), prompts=prompts, calls=calls, options=[])
# The real client hands back a bounded copy. The fake records the bounds and answers as itself,
# so a test can assert what the optional headline call was limited to.
client.with_options = lambda **kwargs: (client.options.append(kwargs), client)[1]
return client
def test_the_headline_call_never_sees_a_merge_the_thread_left_out() -> None:
# The single-call version asked the model not to name a change it had dropped, and shipped two
# headlines that named one anyway: both accurate, both with no line under them and no link. The
# split makes that unreachable rather than forbidden, so what is asserted is the prompt, not the
# answer. A merge dropped for citing no valid rule must be absent from the second call's input.
prs = [
_pr_stub("o/r", 1, "Kept", "https://github.com/o/r/pull/1"),
_pr_stub("o/r", 2, "Dropped and unmentionable", "https://github.com/o/r/pull/2"),
]
# The kept PR's reviewed summary has to travel with it. It is where a condition like a flag
# lives, and a headline that drops one states a gated change as shipped.
prs[0].summary_line = "Playback blocking is behind a flag, off by default."
selection = json.dumps(
{
"prs": [
{"index": 0, "rule": "contract", "summary": "The kept one."},
{"index": 1, "rule": "vibes", "summary": "The dropped one."},
]
}
)
client = _recording_llm_client([selection, json.dumps({"headline": "The kept one shipped."})])
with patch("products.stamphog.backend.logic.digest.build_ai_gateway_anthropic_client", return_value=client):
summary = summarize_merged_prs(prs)
assert [p.pr_number for p in summary.prs] == [1]
assert summary.headline == "The kept one shipped."
selection_prompt, headline_prompt = client.prompts
assert "Dropped and unmentionable" in selection_prompt
assert "Dropped and unmentionable" not in headline_prompt
assert "The kept one." in headline_prompt
assert "Playback blocking is behind a flag, off by default." in headline_prompt
def test_a_headline_failure_keeps_the_judged_digest() -> None:
# The selection call already succeeded, so its lines are the digest. Taking a second-call
# timeout to the deterministic fallback would replace judged lines with unreviewed titles and
# throw away the half of the work that worked.
prs = [_pr_stub("o/r", 1, "Kept", "https://github.com/o/r/pull/1")]
answers = [
json.dumps({"prs": [{"index": 0, "rule": "contract", "summary": "It ships."}]}),
RuntimeError("gateway down"),
]
client = _recording_llm_client(answers)
with patch("products.stamphog.backend.logic.digest.build_ai_gateway_anthropic_client", return_value=client):
summary = summarize_merged_prs(prs)
assert summary.judged is True
assert [p.summary for p in summary.prs] == ["It ships."]
assert summary.headline == ""
# The same call also has to be bounded. It sits in front of a Slack post that is already ready,
# and a team's audiences post one at a time, so an unbounded stall on an optional call holds up
# this digest and every audience behind it. The client's own default is ten minutes with retries.
assert client.options == [{"timeout": _HEADLINE_TIMEOUT_SECONDS, "max_retries": _HEADLINE_MAX_RETRIES}]
# The headline answers in a paragraph, so it reserves far less output than the selection call.
assert client.calls[1]["max_tokens"] == _HEADLINE_MAX_TOKENS < client.calls[0]["max_tokens"]
def test_the_digest_call_names_its_product_team_and_source() -> None:
# The Go gateway serves Claude only on the Messages shape and reads labels from the client's
# headers, so the builder carries the product, team and source tag and every call sets the
# output ceiling the shape requires.
prs = [_pr_stub("o/r", 1, "Kept", "https://github.com/o/r/pull/1")]
client = _recording_llm_client([json.dumps({"prs": [{"index": 0, "rule": "contract", "summary": "It ships."}]})])
with patch(
"products.stamphog.backend.logic.digest.build_ai_gateway_anthropic_client", return_value=client
) as build:
summary = summarize_merged_prs(prs)
team_id = prs[0].team_id
assert summary.judged is True
build.assert_called_once_with(
ai_product="aio_stamphog",
team_id=team_id,
properties={"source_product": "stamphog_digest"},
distinct_id=f"team-{team_id}",
)
(selection_call,) = client.calls[:1]
assert selection_call["model"] == "claude-sonnet-5-5"
assert selection_call["max_tokens"] > 0
assert selection_call["metadata"] == {"user_id": f"team-{team_id}"}
assert "extra_headers" not in selection_call and "user" not in selection_call
def _audience(owned_count: int, reason: AudienceReason = AudienceReason.OWNED) -> PullRequestAudience:
return PullRequestAudience(
audience_key=AUDIENCE,
reason=reason,
owned_files=[f"a{i}.py" for i in range(owned_count)],
owned_file_count=owned_count,
)
@pytest.mark.parametrize(
"owned_count,changed_files,dropped",
[
(1, GRAZE_CHANGED_FILES, True),
(1, GRAZE_CHANGED_FILES - 1, False),
(2, GRAZE_CHANGED_FILES, False),
],
)
def test_a_merge_that_only_grazed_the_team_never_reaches_the_model(
owned_count: int, changed_files: int, dropped: bool
) -> None:
# A repo-wide config change owned one line in ten products and reached every one of their
# channels. The prompt named the graze and asked the model to drop it, and the model kept it
# anyway: the prompt carries no diff, so nothing in it says what that one file does. Code
# decides now. The grazed merge goes first so a filter that forgot to renumber the index map
# would resolve the model's index 0 to the wrong PR.
grazed = _pr_stub("o/r", 1, "Swept in", "https://github.com/o/r/pull/1")
grazed.changed_files = changed_files
owned = _pr_stub("o/r", 2, "Ours", "https://github.com/o/r/pull/2")
audiences = [_audience(owned_count), _audience(1)]
selection = json.dumps(
{"prs": [{"index": 0, "rule": "contract", "scope": SCOPE_YOUR_FILES, "summary": "The first one."}]}
)
client = _recording_llm_client([selection, json.dumps({"headline": "Something shipped."})])
with patch("products.stamphog.backend.logic.digest.build_ai_gateway_anthropic_client", return_value=client):
summary = summarize_merged_prs([grazed, owned], audiences)
assert ("Swept in" in client.prompts[0]) is not dropped
assert [p.pr_number for p in summary.prs] == [2 if dropped else 1]
# The scope line still counts every merge the audience claimed. Counting only what survived
# would tell a swept team that nothing landed in its area at all.
assert summary.considered == 2
@pytest.mark.parametrize(
"audience,claim,kept,headline_reads_title",
[
(_audience(3), {"scope": "whole_pr"}, False, False),
(_audience(3), {}, False, False),
(_audience(3), {"scope": 42}, False, False),
(_audience(3), {"scope": SCOPE_YOUR_FILES}, True, False),
(_audience(8), {}, True, True),
(_audience(0, AudienceReason.REPO_DECLARED), {}, True, True),
],
ids=[
"a_line_about_the_whole_pr_is_not_this_teams_news",
"and_neither_is_one_that_claims_no_perspective",
"nor_one_whose_claim_is_not_even_a_string",
"a_line_written_from_the_owned_files_stays",
"a_team_owning_every_changed_file_claims_nothing",
"and_a_repo_declared_audience_has_no_files_to_claim",
],
)
def test_a_line_about_another_teams_half_of_a_merge_is_dropped(
audience: PullRequestAudience, claim: dict[str, Any], kept: bool, headline_reads_title: bool
) -> None:
# The model names the perspective in its answer, so the code can check it here.
#
# The partly owned merge goes first so a check that dropped by position rather than by the index
# the model was given would take the wrong entry with it.
partly = _pr_stub("o/r", 1, "Adds a facade the other product calls", "https://github.com/o/r/pull/1")
partly.changed_files = 8
ours = _pr_stub("o/r", 2, "Ours", "https://github.com/o/r/pull/2")
ours.changed_files = 2
entry = {"index": 0, "rule": "contract", "summary": "The other team's feature ships.", **claim}
selection = json.dumps(
{
"prs": [
entry,
{"index": 1, "rule": "contract", "scope": SCOPE_YOUR_FILES, "summary": "Our own area changed."},
]
}
)
client = _recording_llm_client([selection, json.dumps({"headline": "Something shipped."})])
with patch("products.stamphog.backend.logic.digest.build_ai_gateway_anthropic_client", return_value=client):
summary = summarize_merged_prs([partly, ours], [audience, _audience(2)])
ours_line = (2, "Our own area changed.")
expected = [(1, "The other team's feature ships."), ours_line] if kept else [ours_line]
assert [(pr.pr_number, pr.summary) for pr in summary.prs] == expected
# The title describes the whole pull request. A headline shown it for a merge the team owns only
# part of wrote about the other team's half while the team's own line was about its files.
headline_prompt = client.prompts[1]
assert ("Adds a facade the other product calls" in headline_prompt) is headline_reads_title
assert "Our own area changed." in headline_prompt
_WHOLE_CHANGE = "Uploads pause when a workspace spends the daily quota."
_OUR_CLAUSE = "the upload path asks a limiter before it writes."
_THEIR_CLAUSE = "@PostHog/team-billing: the counters moved."
_MULTI_TEAM_SUMMARY = f"{_WHOLE_CHANGE} @PostHog/{AUDIENCE}: {_OUR_CLAUSE} {_THEIR_CLAUSE}"
_OTHER_TEAM_SUMMARY = f"{_WHOLE_CHANGE} {_THEIR_CLAUSE}"
@pytest.mark.parametrize(
"summary_line,audience,changed_files,expected",
[
(_MULTI_TEAM_SUMMARY, _audience(3), 8, f"{_WHOLE_CHANGE} {_OUR_CLAUSE}"),
(_OTHER_TEAM_SUMMARY, _audience(2), 2, _WHOLE_CHANGE),
(
_MULTI_TEAM_SUMMARY,
PullRequestAudience(audience_key=f"{REPO_AUDIENCE_PREFIX}{REPO}", reason=AudienceReason.REPO_DECLARED),
8,
_WHOLE_CHANGE,
),
(
f"The CLI moves to @posthog/cli: the old binary is gone. @PostHog/{AUDIENCE}: {_OUR_CLAUSE} {_THEIR_CLAUSE}",
_audience(3),
8,
"",
),
(
f"{_WHOLE_CHANGE[:-1]} @PostHog/{AUDIENCE}: {_OUR_CLAUSE[:-1]} {_THEIR_CLAUSE}",
_audience(3),
8,
"",
),
(
f"{_WHOLE_CHANGE} @PostHog/{AUDIENCE}: {_OUR_CLAUSE} "
f"@PostHog/team-billing: the counters moved. @PostHog/{AUDIENCE}: reads them.",
_audience(3),
8,
f"{_WHOLE_CHANGE} {_OUR_CLAUSE}",
),
],
ids=[
"a_team_owning_part_of_the_merge_reads_its_own_clause",
"a_team_owning_all_of_it_reads_the_whole_change_sentence",
"and_so_does_a_repo_that_declared_its_own_channel",
"a_handle_that_does_not_open_a_sentence_discards_the_summary_rather_than_leaking_it",
"and_so_does_a_clause_written_without_a_period_before_it",
"a_handle_named_again_inside_another_clause_does_not_replace_the_teams_own",
],
)
def test_the_prompt_carries_only_the_reviewed_text_written_for_this_audience(
summary_line: str, audience: PullRequestAudience, changed_files: int, expected: str
) -> None:
# The reviewer holds the diff, so it decides what each team hears and writes them a clause each.
# Picking that clause here, rather than asking the model to ignore the others, is what makes
# another team's half of the merge unreachable: it is not in the prompt to be picked.
pr = _pr_stub("o/r", 1, "Ship it", "https://github.com/o/r/pull/1")
pr.changed_files = changed_files
pr.summary_line = summary_line
prompt = _build_selection_prompt([pr], [audience], _audience_team_slug([audience]))
if expected:
assert f"<reviewed_summary index=0>{expected}</reviewed_summary>" in prompt
else:
assert "<reviewed_summary index=0>" not in prompt
assert "team-billing" not in prompt
assert "the counters moved" not in prompt
def test_a_merge_the_reviewer_wrote_this_team_no_clause_for_never_reaches_the_model() -> None:
# The reviewer named the owning teams and this team was not one of them, so it had the diff and
# found nothing to say about this team's files. Dropping it here costs the team silence; posting
# it costs them the other team's announcement in their own channel.
#
# The dropped merge goes first so a filter that forgot to renumber the index map would resolve
# the model's index 0 to the wrong PR.
unaddressed = _pr_stub("o/r", 1, "Somebody else's half", "https://github.com/o/r/pull/1")
unaddressed.changed_files = 8
unaddressed.summary_line = (
"Invoices show the booking a charge came from. @PostHog/team-billing: the screen reads a facade."
)
ours = _pr_stub("o/r", 2, "Ours", "https://github.com/o/r/pull/2")
ours.changed_files = 2
selection = json.dumps(
{"prs": [{"index": 0, "rule": "contract", "scope": SCOPE_YOUR_FILES, "summary": "Our own area changed."}]}
)
client = _recording_llm_client([selection, json.dumps({"headline": "Something shipped."})])
with patch("products.stamphog.backend.logic.digest.build_ai_gateway_anthropic_client", return_value=client):
summary = summarize_merged_prs([unaddressed, ours], [_audience(3), _audience(2)])
assert "Somebody else's half" not in client.prompts[0]
assert [(pr.pr_number, pr.summary) for pr in summary.prs] == [(2, "Our own area changed.")]
# The scope line still counts every merge the audience claimed, the way the graze rule leaves it.
assert summary.considered == 2
def test_the_selection_prompt_asks_for_the_perspective_the_code_checks() -> None:
# The two halves of this rule sit in different files. The code drops a partly owned merge whose
# line does not claim the team's own files, so a prompt that stopped asking for that claim would
# empty those teams' digests without failing anything.
pr = _pr_stub("o/r", 1, "Ship it", "https://github.com/o/r/pull/1")
pr.changed_files = 8
prompt = _build_selection_prompt([pr], [_audience(3)], AUDIENCE)
assert SCOPE_YOUR_FILES in prompt
assert "whole_pr" in prompt
assert '"scope": "your_files"' in prompt # the shape the answer is read out of
# The audience is the only thing in either prompt that says whose side the line is written
# from, now that the clause it reads is picked in code rather than named in the prompt.
assert f"the team `{AUDIENCE}`" in prompt
def test_the_headline_prompt_asks_for_a_paragraph_every_time() -> None:
# An empty headline on a single-change digest makes the renderer promote that change's own
# line, so the channel post and the first thread line become the same sentence.
picked = [
DigestPRSummary(
pr_number=1,
title="Ship it",
url="https://github.com/o/r/pull/1",
author_login="dev",
summary="The scanner stops at 24 months.",
repository="o/r",
)
]
prompt = _build_headline_prompt(picked, {}, AUDIENCE, frozenset())
assert "empty string" not in prompt
assert "do not restate its line" in prompt
assert f"the team `{AUDIENCE}`" in prompt
def test_a_model_outage_posts_a_short_plain_list_and_says_it_judged_nothing() -> None:
# The fallback keeps merge order and judges nothing, so it needs its own low rail: the bar that
# normally keeps a digest short never runs on this path. It also has to mark itself, because a
# run consumes every merge it claims either way and the post reads like an ordinary quiet day.
prs = [_pr_stub("o/r", n, f"Change {n}", f"https://github.com/o/r/pull/{n}") for n in range(MAX_FALLBACK_PRS + 3)]
with patch(
"products.stamphog.backend.logic.digest.build_ai_gateway_anthropic_client",
side_effect=RuntimeError("gateway down"),
):
summary = summarize_merged_prs(prs)
assert summary.judged is False
assert len(summary.prs) == MAX_FALLBACK_PRS
assert summary.considered == len(prs)
assert summary.headline == ""
@override_settings(
AI_GATEWAY_URL="", AI_GATEWAY_API_KEY="", LLM_GATEWAY_URL="http://llm-gateway.test", LLM_GATEWAY_API_KEY="phx_test"
)
def test_without_the_ai_gateway_the_digest_never_reaches_the_python_gateway() -> None:
# The legacy gateway has no stamphog route, so a process without the Go pair posts the plain list
# even when a Python-gateway client would have answered.
prs = [_pr_stub("o/r", 1, "Kept", "https://github.com/o/r/pull/1")]
client = _recording_llm_client([json.dumps({"prs": [{"index": 0, "rule": "contract", "summary": "It ships."}]})])
with patch("posthog.llm.gateway_client.get_anthropic_gateway_client", return_value=client) as python_gateway:
summary = summarize_merged_prs(prs)
assert summary.judged is False
python_gateway.assert_not_called()
assert client.calls == []
def test_same_pr_number_across_repos_both_survive_summarization() -> None:
# A team digest spans repos, where PR numbers repeat. Keying by bare pr_number collapsed
# acme/a#123 and acme/b#123 into one entry (the dict held one row) and the LLM path could only
# represent one of them. Keying by the assigned index keeps both. If this regresses the code
# falls back to titles, so asserting the LLM summaries survive catches the collision.
prs = [
_pr_stub("acme/a", 123, "A change", "https://github.com/acme/a/pull/123"),
_pr_stub("acme/b", 123, "B change", "https://github.com/acme/b/pull/123"),
]
selection = json.dumps(
{
"prs": [
{"index": 0, "rule": "contract", "summary": "repo a change"},
{"index": 1, "rule": "customer", "summary": "repo b change"},
]
}
)
with patch(
"products.stamphog.backend.logic.digest.build_ai_gateway_anthropic_client",
return_value=_recording_llm_client([selection, json.dumps({"headline": "Both repos changed."})]),
):
summary = summarize_merged_prs(prs)
assert len(summary.prs) == 2
assert {p.url for p in summary.prs} == {
"https://github.com/acme/a/pull/123",
"https://github.com/acme/b/pull/123",
}
assert {p.summary for p in summary.prs} == {"repo a change", "repo b change"}
@pytest.mark.parametrize(
"raw_summary,expected",
[
("The widget opens on the first click.", "The widget opens on the first click."),
(" padded ", "padded"),
(None, "The reviewer's own sentence."),
("", "The reviewer's own sentence."),
({"text": "Shipped"}, "The reviewer's own sentence."),
(["Shipped"], "The reviewer's own sentence."),
(42, "The reviewer's own sentence."),
],
ids=[
"a_string_is_the_line",
"surrounding_whitespace_goes",
"an_omitted_summary_falls_back_to_the_reviewed_sentence",
"an_empty_summary_falls_back_too",
"a_dict_never_reaches_the_post_as_its_repr",
"a_list_does_not_either",
"nor_a_number",
],
)
def test_only_a_string_becomes_a_change_line(raw_summary: Any, expected: str) -> None:
# Coercing whatever arrived put `{'text': 'Shipped'}` in the thread as its Python repr, and the
# channel lead can now promote a change line, so the repr had a route to the one line a reader
# cannot skip. Falling back to the reviewed sentence keeps a malformed entry saying something
# true rather than dropping the change.
pr = _pr_stub("o/r", 1, "The author's own claim.", "https://github.com/o/r/pull/1")
pr.summary_line = "The reviewer's own sentence."
content = json.dumps({"prs": [{"index": 0, "rule": "contract", "summary": raw_summary}]})
picked = _parse_selection(content, {0: pr}, frozenset())
assert [p.summary for p in picked] == [expected]
def test_the_paths_that_skip_the_model_drop_the_other_teams_clauses() -> None:
# The reviewed summary carries one clause per owning team now, and two paths post it without a
# model picking this team's clause out of it: a malformed entry, and the outage fallback.
pr = _pr_stub("o/r", 1, "The author's own claim.", "https://github.com/o/r/pull/1")
pr.summary_line = (
"Uploads pause when a workspace spends its quota. "
"@PostHog/team-storage: the upload path asks a limiter first. "
"@PostHog/team-billing: the counters live in that limiter."
)
whole_change = "Uploads pause when a workspace spends its quota."
content = json.dumps({"prs": [{"index": 0, "rule": "contract", "summary": None}]})
picked = _parse_selection(content, {0: pr}, frozenset())
assert [p.summary for p in picked] == [whole_change]
assert [p.summary for p in _fallback_summary([pr], 1).prs] == [whole_change]
@parameterized.expand(
[
("empty_list_is_intentional_filtering", '{"prs": []}', True),
("unrecognizable_entries_are_not", '{"prs": [{"index": 99}, "junk"]}', False),
("missing_key_is_not", '{"summary": "x"}', False),
# A kept PR must name the rule that admits it. Without that check the model keeps most of a
# routine batch and calls all of it a customer change, which is the drift the rules catch.
# A response naming real merges that cleared no rule was read, so it is a judged empty
# result. Sending it to the fallback would post ten unjudged titles for the one answer that
# broke the bar outright.
("a_kept_pr_without_a_rule_is_a_judged_empty", '{"prs": [{"index": 0, "summary": "x"}]}', True),
("an_invented_rule_is_a_judged_empty", '{"prs": [{"index": 0, "rule": "vibes", "summary": "x"}]}', True),
("an_index_naming_no_merge_is_still_unreadable", '{"prs": [{"index": 99, "rule": "contract"}]}', False),
# `in` against a frozenset raises on an unhashable value, and the raise escaped into the
# outage fallback, which posts unjudged titles for a response that named no valid rule.
("an_unhashable_rule_is_a_judged_empty", '{"prs": [{"index": 0, "rule": [], "summary": "x"}]}', True),
]
)
def test_only_a_genuinely_empty_result_posts_nothing(_name: str, content: str, accepted: bool) -> None:
# Keeping nothing is a real answer for an owning team. A list we could read no PR out of is a
# broken response wearing that shape, and accepting it would consume every claimed audience for
# an empty post instead of falling back to the deterministic list.
prs_by_index = {0: _pr_stub("PostHog/posthog", 1, "Title", "https://example.com/1")}
if accepted:
assert _parse_selection(content, prs_by_index, frozenset()) == []
else:
with pytest.raises(ValueError):
_parse_selection(content, prs_by_index, frozenset())
@pytest.mark.parametrize(
"raw_headline,expected",
[
("The scanner stops at 24 months.", "The scanner stops at 24 months."),
(" The scanner stops.\n\n It also logs. ", "The scanner stops. It also logs."),
("- The scanner stops.\n- It also logs.", "- The scanner stops. - It also logs."),
("See https://github.com/o/r/pull/1 for the change.", ""),
("<https://github.com/o/r/pull/1|The scanner stops.>", ""),
(["not", "a", "string"], ""),
],
ids=[
"a_plain_paragraph_survives",
"line_breaks_collapse_into_one_paragraph",
"a_list_collapses_rather_than_reaching_the_channel_as_lines",
"a_bare_url_drops_the_whole_headline",
"a_slack_link_drops_the_whole_headline",
"a_non_string_drops_the_whole_headline",
],
)
def test_the_headline_reaches_the_channel_as_one_link_free_paragraph(raw_headline: Any, expected: str) -> None:
# The headline is the only part posted where a reader cannot choose to skip it, and it is meant
# to read as prose. A model that answers with bullets puts the list back in the channel, and one
# that answers with a URL either shows a raw link mid-sentence or, once escaped, shows raw
# markup. Neither is repairable in place, so a link drops the headline and the renderer leads
# with the change's own line instead.
contents = [
json.dumps({"prs": [{"index": 0, "rule": "contract", "summary": "Ship it."}]}),
json.dumps({"headline": raw_headline}),
]
with patch(
"products.stamphog.backend.logic.digest.build_ai_gateway_anthropic_client",
return_value=_recording_llm_client(contents),
):
summary = summarize_merged_prs([_pr_stub("o/r", 1, "Ship it", "https://example.com/1")])
assert summary.headline == expected
# A rejected headline never costs the change line the selection call already wrote.
assert len(summary.prs) == 1
def _slack_destination(team: Any) -> Destination:
integration = Integration.objects.create(
team_id=team.id, kind="slack", config={}, sensitive_config={"access_token": "x"}
)
return Destination(
slack_integration_id=integration.id,
channel_id="C1",
channel_name="team-devex",
source=ChannelResolutionSource.SLACK_NAME_MATCH,
)
def _one_pr_summary(headline: str = "") -> DigestSummary:
return DigestSummary(
considered=1,
headline=headline,
prs=[
DigestPRSummary(
pr_number=1,
title="Add util helper",
url="https://github.com/acme/widgets/pull/1",
author_login="devex-dev",
summary="Add util helper",
repository=REPO,
)
],
)
@pytest.mark.django_db(databases=PRODUCT_DATABASES)
def test_the_changes_are_posted_as_a_thread_reply_under_the_lead(team) -> None:
# The channel gets one line and the change lines hang off it. Losing thread_ts posts those
# lines as a second top-level message, so the channel carries more of the digest than the flat
# version it replaced rather than less.
destination = _slack_destination(team)
FakeSlackIntegration.reset(channels=[])
with patch("products.stamphog.backend.logic.slack_digest.SlackIntegration", FakeSlackIntegration):
summary = _one_pr_summary("The util helper landed.")
message_ts = post_digest_lead(team.id, destination, summary)
assert message_ts == "1234.5678"
post_digest_details(team.id, destination, summary, message_ts)
posted = FakeSlackIntegration.posted_messages
assert [p["thread_ts"] for p in posted] == [None, "1234.5678"]
assert "pull/1" not in str(posted[0]["blocks"])
assert "pull/1" in str(posted[1]["blocks"])
@pytest.mark.django_db(databases=PRODUCT_DATABASES)
def test_a_failed_thread_reply_still_counts_as_a_posted_digest(team) -> None:
# Slack already accepted the lead, and the caller writes that ts as proof-of-post before
# consuming the claimed PRs. Raising here would mark the run failed, unlink its PRs, and post
# the same lead into the channel again tomorrow.
destination = _slack_destination(team)
FakeSlackIntegration.reset(channels=[], fail_thread_replies=True)
with patch("products.stamphog.backend.logic.slack_digest.SlackIntegration", FakeSlackIntegration):
summary = _one_pr_summary("The util helper landed.")
message_ts = post_digest_lead(team.id, destination, summary)
assert message_ts == "1234.5678"
post_digest_details(team.id, destination, summary, message_ts)
assert len(FakeSlackIntegration.posted_messages) == 2
def _slack_stub(post_error: str, join_error: str | None, joined: list[str]) -> MagicMock:
"""SlackIntegration stand-in whose post fails with ``post_error`` until the app has joined."""
stub = MagicMock()
def post(**kwargs: Any) -> dict[str, Any]:
if not joined:
raise SlackApiError(post_error, {"ok": False, "error": post_error})
return {"ok": True, "ts": "9999.1"}
def join(channel: str) -> dict[str, Any]:
if join_error:
# already_in_channel means somebody else already put the app in there, so the retried
# post has to succeed — the stub records the membership before raising.
if join_error == "already_in_channel":
joined.append(channel)
raise SlackApiError(join_error, {"ok": False, "error": join_error})
joined.append(channel)
return {"ok": True}
stub.client.chat_postMessage.side_effect = post
stub.client.conversations_join.side_effect = join
return stub
@pytest.mark.parametrize(
"post_error,join_error,expected_error,joined",
[
("not_in_channel", None, None, ["C1"]),
("not_in_channel", "already_in_channel", None, ["C1"]),
("not_in_channel", "missing_scope", DigestSlackError, []),
("channel_not_found", None, SlackApiError, []),
],
ids=[
"joins_then_posts",
"already_in_channel_counts_as_joined",
"refused_join_names_the_reason_and_the_invite",
"other_slack_errors_propagate",
],
)
@pytest.mark.django_db(databases=PRODUCT_DATABASES)
def test_post_digest_joins_a_channel_the_app_was_never_invited_to(
team, post_error: str, join_error: str | None, expected_error: type[Exception] | None, joined: list[str]
) -> None:
# An auto-provisioned channel is matched off the workspace list, so the app is not a member of it
# and the first post comes back not_in_channel. Joining is what makes that post land, and a
# concurrent worker joining first (already_in_channel) must not fail a digest whose retry would
# have gone through. A genuine refusal names Slack's reason and the invite, because neither is
# derivable from an error code by the person reading the run.
destination = _slack_destination(team)
summary = _one_pr_summary()
actually_joined: list[str] = []
stub = _slack_stub(post_error, join_error, actually_joined)
with patch("products.stamphog.backend.logic.slack_digest.SlackIntegration", return_value=stub):
if expected_error is None:
assert post_digest_lead(team.id, destination, summary) == "9999.1"
else:
with pytest.raises(expected_error) as caught:
post_digest_lead(team.id, destination, summary)
if expected_error is DigestSlackError:
assert "/invite @PostHog" in str(caught.value)
assert str(join_error) in str(caught.value)
assert actually_joined == joined