:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(review-hog): ask for a rethink after three review rounds (#119856)
Georges-Antoine Assi04b25b8e5fa0
diff --git a/products/review_hog/ARCHITECTURE.md b/products/review_hog/ARCHITECTURE.mdindex c4ce280e6dc3..5b652aca0e10 100644--- a/products/review_hog/ARCHITECTURE.md+++ b/products/review_hog/ARCHITECTURE.md@@ -364,6 +364,13 @@ everything would otherwise refill it. Pull requests split into two arms by a has(`budget_arm`). Only the `budget` arm applies the budget, and `finding_budget_arm` on `reviewhog_review_completed` and`reviewhog_finding_outcome` splits the measurement.+The same module counts review rounds (`ReviewRound`): the turns that published on the pull request before this one, by+any mode and trigger, plus one. From round `ROUND_CAP` (3) on, a review that posts findings ends with a line that names+the round and asks the author to reconsider the approach, because an author and a reviewer that fix one finding at a+time loop, and most of what a late round flags was already there in the first. A clean turn carries no note. Every+`reviewhog_review_completed` event carries `review_round`, so convergence (how many rounds a pull request takes before+a clean turn) reads from the events.+Every finding the turn drops after scope cleaning persists as a `dropped_finding` artefact (`DroppedFindingArtefact`,written by `replace_dropped_findings` beside `replace_deduplicated_findings` in the dedup activity): the full findingwith its `reported_priority`, the session's pass and part, the `disposition` (`old_code`, `dedup_prior`, `dedup_comment`,diff --git a/products/review_hog/backend/reviewer/finding_budget.py b/products/review_hog/backend/reviewer/finding_budget.pyindex e9c9809cb4a3..2a50079b061e 100644--- a/products/review_hog/backend/reviewer/finding_budget.py+++ b/products/review_hog/backend/reviewer/finding_budget.py@@ -29,6 +29,7 @@ from products.review_hog.backend.reviewer.constants import effective_priority, pfrom products.review_hog.backend.reviewer.models.issues_review import IssuePriorityFINDING_BUDGET = 20+ROUND_CAP = 3BudgetArm = Literal["budget", "control"]@@ -78,6 +79,39 @@ class FindingBudget:return max(FINDING_BUDGET - self.posted_before, 0)+@frozen+class ReviewRound:+ """Which review round a turn is on its pull request: the turns that published before it, plus one.++ A round is a turn that posted to GitHub, by any mode and trigger, the same count the finding budget+ uses. A turn that kept nothing is not a round, so a clean turn repeats the number of the next one.+ """++ number: int++ @classmethod+ def for_turn(cls, report: ReviewReport, *, run_index: int) -> "ReviewRound":+ published_before = sum(1 for index in (report.published_head_shas or {}) if int(index) < run_index)+ return cls(number=published_before + 1)++ @property+ def asks_for_a_rethink(self) -> bool:+ return self.number >= ROUND_CAP+++def round_note(review_round: ReviewRound) -> str:+ """The line a review ends with from round `ROUND_CAP` on, so the author reconsiders the approach.++ An author and a reviewer that fix and re-review one finding at a time can loop for a long time, and most+ of what a late round flags was already there in the first one.+ """+ return (+ f"This is review round {review_round.number} on this pull request. When findings keep coming back in the "+ "same code, the approach is usually the problem. Before the next fix, check whether a different design "+ "would remove the whole class of findings."+ )++def _posted_count(report: ReviewReport, findings: Sequence[tuple[ReviewIssueFinding, ValidationVerdict | None]]) -> int:"""The valid findings that published turns let through their urgency threshold.diff --git a/products/review_hog/backend/reviewer/tests/test_finding_budget.py b/products/review_hog/backend/reviewer/tests/test_finding_budget.pyindex 32094c3f6999..62aa53383618 100644--- a/products/review_hog/backend/reviewer/tests/test_finding_budget.py+++ b/products/review_hog/backend/reviewer/tests/test_finding_budget.py@@ -2,7 +2,15 @@ import pytestfrom products.review_hog.backend.models import ReviewReportfrom products.review_hog.backend.reviewer.artefact_content import ReviewIssueFinding, ValidationVerdict-from products.review_hog.backend.reviewer.finding_budget import FINDING_BUDGET, FindingBudget, budget_arm, budget_note+from products.review_hog.backend.reviewer.finding_budget import (+ FINDING_BUDGET,+ ROUND_CAP,+ FindingBudget,+ ReviewRound,+ budget_arm,+ budget_note,+ round_note,+)from products.review_hog.backend.reviewer.models.issues_review import IssuePriority@@ -62,3 +70,31 @@ def test_budget_note_offers_only_a_review_that_runs_on_this_head(comment_commandassert note.startswith("This pull request reached its limit of 20 automatic review findings")assert ("`@posthog review --deep`" in note) is offers_deepassert "`@posthog review` " not in note+++ "published_head_shas,run_index,expected_number,asks_for_a_rethink",+ [+ (None, 1, 1, False),+ # Turn 2 kept nothing, so turn 3 is still round 2.+ ({"1": "a"}, 3, 2, False),+ ({"1": "a", "2": "b"}, 3, 3, True),+ # Read after this turn published: its own key does not count twice.+ ({"1": "a", "2": "b", "3": "c"}, 3, 3, True),+ ({"1": "a", "2": "b", "3": "c", "5": "e"}, 6, 5, True),+ ],+)+def test_review_round_counts_the_turns_that_published_before_this_one(+ published_head_shas: dict[str, str] | None, run_index: int, expected_number: int, asks_for_a_rethink: bool+) -> None:+ report = ReviewReport(repository="PostHog/posthog", pr_number=1, published_head_shas=published_head_shas)++ review_round = ReviewRound.for_turn(report, run_index=run_index)++ assert (review_round.number, review_round.asks_for_a_rethink) == (expected_number, asks_for_a_rethink)+++def test_round_note_names_the_round() -> None:+ assert round_note(ReviewRound(number=ROUND_CAP)).startswith(+ f"This is review round {ROUND_CAP} on this pull request."+ )diff --git a/products/review_hog/backend/reviewer/tests/test_publish_review.py b/products/review_hog/backend/reviewer/tests/test_publish_review.pyindex cb77c774a78f..5348f8c78e87 100644--- a/products/review_hog/backend/reviewer/tests/test_publish_review.py+++ b/products/review_hog/backend/reviewer/tests/test_publish_review.py@@ -601,11 +601,12 @@ class TestPublishReviewGate:published_priorities=_SHOULD_FIX_PUBLISHED,review_mode="flash",budget_note="Budget note.",+ round_note="Round note.",)assert outcome.posted is expected_postedif expected_posted:- assert "Budget note." in mock_post.call_args.args[3]+ assert "Budget note.\n\nRound note.\n\n" in mock_post.call_args.args[3]else:mock_post.assert_not_called()diff --git a/products/review_hog/backend/reviewer/tools/publish_review.py b/products/review_hog/backend/reviewer/tools/publish_review.pyindex 2ba13e33ae75..eb6f5ac297bf 100644--- a/products/review_hog/backend/reviewer/tools/publish_review.py+++ b/products/review_hog/backend/reviewer/tools/publish_review.py@@ -21,7 +21,7 @@ from products.review_hog.backend.reviewer.constants import (published_priorities_for,)from products.review_hog.backend.reviewer.diff_position import build_diff_line_map, find_diff_position-from products.review_hog.backend.reviewer.finding_budget import budget_note+from products.review_hog.backend.reviewer.finding_budget import ReviewRound, budget_note, round_notefrom products.review_hog.backend.reviewer.models.github_meta import PRFilefrom products.review_hog.backend.reviewer.models.issues_review import IssuePriorityfrom products.review_hog.backend.reviewer.persistence import (@@ -131,6 +131,7 @@ def publish_persisted_review()if dropped.disposition == "budget")+ review_round = ReviewRound.for_turn(report, run_index=run_index)outcome = publish_review(owner=owner,repo=repo,@@ -149,6 +150,7 @@ def publish_persisted_review(findings_footer=draft_notice if snapshot is not None and snapshot.pr_metadata.draft else "",answers_request=answers_request,budget_note=budget_note(over_budget, comment_commands_on) if over_budget else "",+ round_note=round_note(review_round) if review_round.asks_for_a_rethink else "",)if outcome.posted:if report.outcomes_emitted_at is not None:@@ -253,6 +255,7 @@ def publish_review(footer: str = "",findings_footer: str = "",budget_note: str = "",+ round_note: str = "",answers_request: bool = False,) -> PublishOutcome:"""Publish the review to GitHub: the stored body plus inline comments from the durable rows.@@ -275,6 +278,9 @@ def publish_review(`budget_note` says the finding budget left findings out. An automatic turn that keeps nothing stays silent: oncethe budget is spent an automatic review only looks for must-fix findings.+ `round_note` tells the author which review round this is and to reconsider the approach. Like the budget note it+ follows findings only, because a clean turn is not a round.+`answers_request` is set when a person asked for the turn, with the label, a comment, or the UI. A cleanStandard turn then posts a short review that says it found nothing. A Deep turn has its status comment for that."""@@ -287,7 +293,7 @@ def publish_review(# `footer` closes the review whichever body posts, so the inline-only body keeps it too.footer_block = f"{footer}\n\n" if footer else ""note_block = f"{large_pr_note(large_pr_parts)}\n\n" if large_pr_parts is not None else ""- budget_block = f"{budget_note}\n\n" if budget_note else ""+ budget_block = "".join(f"{note}\n\n" for note in (budget_note, round_note) if note)valid_findings = load_valid_findings(team_id=team_id, report_id=report_id, run_index=run_index)diff_lines = build_diff_line_map(pr_files)diff --git a/products/review_hog/backend/temporal/activities.py b/products/review_hog/backend/temporal/activities.pyindex a8170c54f4c0..c6f3daade3d7 100644--- a/products/review_hog/backend/temporal/activities.py+++ b/products/review_hog/backend/temporal/activities.py@@ -72,7 +72,7 @@ from products.review_hog.backend.reviewer.constants import (validation_arm_for_mode,)from products.review_hog.backend.reviewer.feature_flags import flash_pipeline_kill_switch_on, standard_coordinator_on-from products.review_hog.backend.reviewer.finding_budget import FindingBudget, budget_arm+from products.review_hog.backend.reviewer.finding_budget import FindingBudget, ReviewRound, budget_armfrom products.review_hog.backend.reviewer.fingerprint import ReviewHogMarker, record_turn_markerfrom products.review_hog.backend.reviewer.lazy_seed import (sync_canonical_authoring,@@ -2492,6 +2492,7 @@ def _track_review_completed(input: TrackReviewCompletedInput) -> None:"pr_draft": _pr_draft(snapshot),# Split by this to measure the finding budget against pull requests that keep posting."finding_budget_arm": budget_arm(report.repository, report.pr_number),+ "review_round": ReviewRound.for_turn(report, run_index=input.run_index).number,"duration_seconds": duration_seconds,# No marker means a turn started before the marker shipped, or a failed marker: the version is unknown."reviewhog_version": input.marker.version if input.marker is not None else None,diff --git a/products/review_hog/backend/tests/test_publish_idempotency.py b/products/review_hog/backend/tests/test_publish_idempotency.pyindex 0274d7a2abbb..49be26b6b16f 100644--- a/products/review_hog/backend/tests/test_publish_idempotency.py+++ b/products/review_hog/backend/tests/test_publish_idempotency.py@@ -259,6 +259,32 @@ class TestPublishIdempotency(BaseTest):"2 lower-priority findings. Must-fix findings still post.")+ @parameterized.expand(+ [+ ("second_round_stays_quiet", {"1": "sha0"}, ""),+ (+ "third_round_asks_for_a_rethink",+ {"1": "sha0", "2": "sha0"},+ "This is review round 3 on this pull request.",+ ),+ ]+ )+ @patch(_PUBLISH)+ @patch(_SNAPSHOT, return_value=None)+ @patch(_AUTH, return_value=("tok", None))+ def test_a_review_says_which_round_it_is_from_the_round_cap_on(+ self, _name, published_head_shas, expected_prefix, _auth, _snapshot, mock_publish+ ) -> None:+ mock_publish.return_value = PublishOutcome(posted=True)+ report_id = self._report()+ ReviewReport.objects.for_team(self.team.id).filter(id=report_id).update(published_head_shas=published_head_shas)++ _publish(replace(_publish_input(self.team.id, report_id, "sha1"), run_index=3))++ note = mock_publish.call_args.kwargs["round_note"]+ assert note[: len(expected_prefix)] == expected_prefix+ assert bool(note) is bool(expected_prefix)+@patch(_PUBLISH)@patch(_SNAPSHOT, return_value=None)@patch(_AUTH, return_value=("tok", None))diff --git a/products/review_hog/backend/tests/test_review_completed_tracking.py b/products/review_hog/backend/tests/test_review_completed_tracking.pyindex d48cb013dbea..894d8f095155 100644--- a/products/review_hog/backend/tests/test_review_completed_tracking.py+++ b/products/review_hog/backend/tests/test_review_completed_tracking.py@@ -209,6 +209,7 @@ class TestTrackReviewCompleted(BaseTest):assert props["pr_draft"] is True# The finding budget's effect is measured by splitting on this, so it must be on every turn.assert props["finding_budget_arm"] == budget_arm("o/r", 7)+ assert props["review_round"] == 1assert 90 <= props["duration_seconds"] < 600assert props["reviewhog_version"] == "reviewhog-full-9-9"assert props["reviewhog_fingerprint"] == "abc1234"