Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(review-hog): ask for a rethink after three review rounds (#119856)

Georges-Antoine Assi04b25b8e5fa0
diff --git a/products/review_hog/ARCHITECTURE.md b/products/review_hog/ARCHITECTURE.md
index c4ce280e6dc3..5b652aca0e10 100644
--- a/products/review_hog/ARCHITECTURE.md
+++ b/products/review_hog/ARCHITECTURE.md
@@ -364,6 +364,13 @@ everything would otherwise refill it. Pull requests split into two arms by a has
(`budget_arm`). Only the `budget` arm applies the budget, and `finding_budget_arm` on `reviewhog_review_completed` and
`reviewhog_finding_outcome` splits the measurement.
+The same module counts review rounds (`ReviewRound`): the turns that published on the pull request before this one, by
+any mode and trigger, plus one. From round `ROUND_CAP` (3) on, a review that posts findings ends with a line that names
+the round and asks the author to reconsider the approach, because an author and a reviewer that fix one finding at a
+time loop, and most of what a late round flags was already there in the first. A clean turn carries no note. Every
+`reviewhog_review_completed` event carries `review_round`, so convergence (how many rounds a pull request takes before
+a clean turn) reads from the events.
+
Every finding the turn drops after scope cleaning persists as a `dropped_finding` artefact (`DroppedFindingArtefact`,
written by `replace_dropped_findings` beside `replace_deduplicated_findings` in the dedup activity): the full finding
with its `reported_priority`, the session's pass and part, the `disposition` (`old_code`, `dedup_prior`, `dedup_comment`,
diff --git a/products/review_hog/backend/reviewer/finding_budget.py b/products/review_hog/backend/reviewer/finding_budget.py
index e9c9809cb4a3..2a50079b061e 100644
--- a/products/review_hog/backend/reviewer/finding_budget.py
+++ b/products/review_hog/backend/reviewer/finding_budget.py
@@ -29,6 +29,7 @@ from products.review_hog.backend.reviewer.constants import effective_priority, p
from products.review_hog.backend.reviewer.models.issues_review import IssuePriority
FINDING_BUDGET = 20
+ROUND_CAP = 3
BudgetArm = Literal["budget", "control"]
@@ -78,6 +79,39 @@ class FindingBudget:
return max(FINDING_BUDGET - self.posted_before, 0)
+@frozen
+class ReviewRound:
+ """Which review round a turn is on its pull request: the turns that published before it, plus one.
+
+ A round is a turn that posted to GitHub, by any mode and trigger, the same count the finding budget
+ uses. A turn that kept nothing is not a round, so a clean turn repeats the number of the next one.
+ """
+
+ number: int
+
+ @classmethod
+ def for_turn(cls, report: ReviewReport, *, run_index: int) -> "ReviewRound":
+ published_before = sum(1 for index in (report.published_head_shas or {}) if int(index) < run_index)
+ return cls(number=published_before + 1)
+
+ @property
+ def asks_for_a_rethink(self) -> bool:
+ return self.number >= ROUND_CAP
+
+
+def round_note(review_round: ReviewRound) -> str:
+ """The line a review ends with from round `ROUND_CAP` on, so the author reconsiders the approach.
+
+ An author and a reviewer that fix and re-review one finding at a time can loop for a long time, and most
+ of what a late round flags was already there in the first one.
+ """
+ return (
+ f"This is review round {review_round.number} on this pull request. When findings keep coming back in the "
+ "same code, the approach is usually the problem. Before the next fix, check whether a different design "
+ "would remove the whole class of findings."
+ )
+
+
def _posted_count(report: ReviewReport, findings: Sequence[tuple[ReviewIssueFinding, ValidationVerdict | None]]) -> int:
"""The valid findings that published turns let through their urgency threshold.
diff --git a/products/review_hog/backend/reviewer/tests/test_finding_budget.py b/products/review_hog/backend/reviewer/tests/test_finding_budget.py
index 32094c3f6999..62aa53383618 100644
--- a/products/review_hog/backend/reviewer/tests/test_finding_budget.py
+++ b/products/review_hog/backend/reviewer/tests/test_finding_budget.py
@@ -2,7 +2,15 @@ import pytest
from products.review_hog.backend.models import ReviewReport
from products.review_hog.backend.reviewer.artefact_content import ReviewIssueFinding, ValidationVerdict
-from products.review_hog.backend.reviewer.finding_budget import FINDING_BUDGET, FindingBudget, budget_arm, budget_note
+from products.review_hog.backend.reviewer.finding_budget import (
+ FINDING_BUDGET,
+ ROUND_CAP,
+ FindingBudget,
+ ReviewRound,
+ budget_arm,
+ budget_note,
+ round_note,
+)
from products.review_hog.backend.reviewer.models.issues_review import IssuePriority
@@ -62,3 +70,31 @@ def test_budget_note_offers_only_a_review_that_runs_on_this_head(comment_command
assert note.startswith("This pull request reached its limit of 20 automatic review findings")
assert ("`@posthog review --deep`" in note) is offers_deep
assert "`@posthog review` " not in note
+
+
+ "published_head_shas,run_index,expected_number,asks_for_a_rethink",
+ [
+ (None, 1, 1, False),
+ # Turn 2 kept nothing, so turn 3 is still round 2.
+ ({"1": "a"}, 3, 2, False),
+ ({"1": "a", "2": "b"}, 3, 3, True),
+ # Read after this turn published: its own key does not count twice.
+ ({"1": "a", "2": "b", "3": "c"}, 3, 3, True),
+ ({"1": "a", "2": "b", "3": "c", "5": "e"}, 6, 5, True),
+ ],
+)
+def test_review_round_counts_the_turns_that_published_before_this_one(
+ published_head_shas: dict[str, str] | None, run_index: int, expected_number: int, asks_for_a_rethink: bool
+) -> None:
+ report = ReviewReport(repository="PostHog/posthog", pr_number=1, published_head_shas=published_head_shas)
+
+ review_round = ReviewRound.for_turn(report, run_index=run_index)
+
+ assert (review_round.number, review_round.asks_for_a_rethink) == (expected_number, asks_for_a_rethink)
+
+
+def test_round_note_names_the_round() -> None:
+ assert round_note(ReviewRound(number=ROUND_CAP)).startswith(
+ f"This is review round {ROUND_CAP} on this pull request."
+ )
diff --git a/products/review_hog/backend/reviewer/tests/test_publish_review.py b/products/review_hog/backend/reviewer/tests/test_publish_review.py
index cb77c774a78f..5348f8c78e87 100644
--- a/products/review_hog/backend/reviewer/tests/test_publish_review.py
+++ b/products/review_hog/backend/reviewer/tests/test_publish_review.py
@@ -601,11 +601,12 @@ class TestPublishReviewGate:
published_priorities=_SHOULD_FIX_PUBLISHED,
review_mode="flash",
budget_note="Budget note.",
+ round_note="Round note.",
)
assert outcome.posted is expected_posted
if expected_posted:
- assert "Budget note." in mock_post.call_args.args[3]
+ assert "Budget note.\n\nRound note.\n\n" in mock_post.call_args.args[3]
else:
mock_post.assert_not_called()
diff --git a/products/review_hog/backend/reviewer/tools/publish_review.py b/products/review_hog/backend/reviewer/tools/publish_review.py
index 2ba13e33ae75..eb6f5ac297bf 100644
--- a/products/review_hog/backend/reviewer/tools/publish_review.py
+++ b/products/review_hog/backend/reviewer/tools/publish_review.py
@@ -21,7 +21,7 @@ from products.review_hog.backend.reviewer.constants import (
published_priorities_for,
)
from products.review_hog.backend.reviewer.diff_position import build_diff_line_map, find_diff_position
-from products.review_hog.backend.reviewer.finding_budget import budget_note
+from products.review_hog.backend.reviewer.finding_budget import ReviewRound, budget_note, round_note
from products.review_hog.backend.reviewer.models.github_meta import PRFile
from products.review_hog.backend.reviewer.models.issues_review import IssuePriority
from products.review_hog.backend.reviewer.persistence import (
@@ -131,6 +131,7 @@ def publish_persisted_review(
)
if dropped.disposition == "budget"
)
+ review_round = ReviewRound.for_turn(report, run_index=run_index)
outcome = publish_review(
owner=owner,
repo=repo,
@@ -149,6 +150,7 @@ def publish_persisted_review(
findings_footer=draft_notice if snapshot is not None and snapshot.pr_metadata.draft else "",
answers_request=answers_request,
budget_note=budget_note(over_budget, comment_commands_on) if over_budget else "",
+ round_note=round_note(review_round) if review_round.asks_for_a_rethink else "",
)
if outcome.posted:
if report.outcomes_emitted_at is not None:
@@ -253,6 +255,7 @@ def publish_review(
footer: str = "",
findings_footer: str = "",
budget_note: str = "",
+ round_note: str = "",
answers_request: bool = False,
) -> PublishOutcome:
"""Publish the review to GitHub: the stored body plus inline comments from the durable rows.
@@ -275,6 +278,9 @@ def publish_review(
`budget_note` says the finding budget left findings out. An automatic turn that keeps nothing stays silent: once
the budget is spent an automatic review only looks for must-fix findings.
+ `round_note` tells the author which review round this is and to reconsider the approach. Like the budget note it
+ follows findings only, because a clean turn is not a round.
+
`answers_request` is set when a person asked for the turn, with the label, a comment, or the UI. A clean
Standard turn then posts a short review that says it found nothing. A Deep turn has its status comment for that.
"""
@@ -287,7 +293,7 @@ def publish_review(
# `footer` closes the review whichever body posts, so the inline-only body keeps it too.
footer_block = f"{footer}\n\n" if footer else ""
note_block = f"{large_pr_note(large_pr_parts)}\n\n" if large_pr_parts is not None else ""
- budget_block = f"{budget_note}\n\n" if budget_note else ""
+ budget_block = "".join(f"{note}\n\n" for note in (budget_note, round_note) if note)
valid_findings = load_valid_findings(team_id=team_id, report_id=report_id, run_index=run_index)
diff_lines = build_diff_line_map(pr_files)
diff --git a/products/review_hog/backend/temporal/activities.py b/products/review_hog/backend/temporal/activities.py
index a8170c54f4c0..c6f3daade3d7 100644
--- a/products/review_hog/backend/temporal/activities.py
+++ b/products/review_hog/backend/temporal/activities.py
@@ -72,7 +72,7 @@ from products.review_hog.backend.reviewer.constants import (
validation_arm_for_mode,
)
from products.review_hog.backend.reviewer.feature_flags import flash_pipeline_kill_switch_on, standard_coordinator_on
-from products.review_hog.backend.reviewer.finding_budget import FindingBudget, budget_arm
+from products.review_hog.backend.reviewer.finding_budget import FindingBudget, ReviewRound, budget_arm
from products.review_hog.backend.reviewer.fingerprint import ReviewHogMarker, record_turn_marker
from products.review_hog.backend.reviewer.lazy_seed import (
sync_canonical_authoring,
@@ -2492,6 +2492,7 @@ def _track_review_completed(input: TrackReviewCompletedInput) -> None:
"pr_draft": _pr_draft(snapshot),
# Split by this to measure the finding budget against pull requests that keep posting.
"finding_budget_arm": budget_arm(report.repository, report.pr_number),
+ "review_round": ReviewRound.for_turn(report, run_index=input.run_index).number,
"duration_seconds": duration_seconds,
# No marker means a turn started before the marker shipped, or a failed marker: the version is unknown.
"reviewhog_version": input.marker.version if input.marker is not None else None,
diff --git a/products/review_hog/backend/tests/test_publish_idempotency.py b/products/review_hog/backend/tests/test_publish_idempotency.py
index 0274d7a2abbb..49be26b6b16f 100644
--- a/products/review_hog/backend/tests/test_publish_idempotency.py
+++ b/products/review_hog/backend/tests/test_publish_idempotency.py
@@ -259,6 +259,32 @@ class TestPublishIdempotency(BaseTest):
"2 lower-priority findings. Must-fix findings still post."
)
+ @parameterized.expand(
+ [
+ ("second_round_stays_quiet", {"1": "sha0"}, ""),
+ (
+ "third_round_asks_for_a_rethink",
+ {"1": "sha0", "2": "sha0"},
+ "This is review round 3 on this pull request.",
+ ),
+ ]
+ )
+ @patch(_PUBLISH)
+ @patch(_SNAPSHOT, return_value=None)
+ @patch(_AUTH, return_value=("tok", None))
+ def test_a_review_says_which_round_it_is_from_the_round_cap_on(
+ self, _name, published_head_shas, expected_prefix, _auth, _snapshot, mock_publish
+ ) -> None:
+ mock_publish.return_value = PublishOutcome(posted=True)
+ report_id = self._report()
+ ReviewReport.objects.for_team(self.team.id).filter(id=report_id).update(published_head_shas=published_head_shas)
+
+ _publish(replace(_publish_input(self.team.id, report_id, "sha1"), run_index=3))
+
+ note = mock_publish.call_args.kwargs["round_note"]
+ assert note[: len(expected_prefix)] == expected_prefix
+ assert bool(note) is bool(expected_prefix)
+
@patch(_PUBLISH)
@patch(_SNAPSHOT, return_value=None)
@patch(_AUTH, return_value=("tok", None))
diff --git a/products/review_hog/backend/tests/test_review_completed_tracking.py b/products/review_hog/backend/tests/test_review_completed_tracking.py
index d48cb013dbea..894d8f095155 100644
--- a/products/review_hog/backend/tests/test_review_completed_tracking.py
+++ b/products/review_hog/backend/tests/test_review_completed_tracking.py
@@ -209,6 +209,7 @@ class TestTrackReviewCompleted(BaseTest):
assert props["pr_draft"] is True
# The finding budget's effect is measured by splitting on this, so it must be on every turn.
assert props["finding_budget_arm"] == budget_arm("o/r", 7)
+ assert props["review_round"] == 1
assert 90 <= props["duration_seconds"] < 600
assert props["reviewhog_version"] == "reviewhog-full-9-9"
assert props["reviewhog_fingerprint"] == "abc1234"