Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(signals): recover failed first pr assignment delivery with bounded retries (#122460)

Dylan Martine1c19eaff1ad
diff --git a/docs/internal/signals-pr-lifecycle.md b/docs/internal/signals-pr-lifecycle.md
index d416dd980250..6c6a2d8a3022 100644
--- a/docs/internal/signals-pr-lifecycle.md
+++ b/docs/internal/signals-pr-lifecycle.md
@@ -339,11 +339,21 @@ If no saved recipient qualifies, the PR remains unassigned.
The personal setting does not remove anyone from the report. Claims and owning teams do not change the selected list or its order.
The report remains the source of the assignment. Logs record unavailable accounts, permission failures, and GitHub delivery failures.
-The worker rechecks the report state, personal settings, account links, project access, GitHub state, and delivery controls before assigning.
+A transient GitHub failure during first assignment, or a first write whose outcome is unknown, retries up to five times.
+The normal delays are 1, 2, 4, 8, and 15 minutes. A longer GitHub rate-limit delay takes precedence.
+Every attempt rechecks the report state, personal settings, account links, project access, GitHub state, and delivery controls.
+An uncertain write that reached GitHub appears as an assignment with an unknown source, so the worker does not repeat it.
+Successful first assignments record owner provenance for later reassessment.
+A permanent GitHub response, a closed pull request, a manual or unknown-source assignment, and a disabled flag end the attempt without retry.
+Permission checks follow the same rule: a non-rate-limit 403 stops recovery; network failures, 429 responses, and server errors permit retries.
+Safe assignment skips still allow review requests under the current review policy. Review requests, later owner reassessment, and legacy assignment policies remain best effort.
+Enabling a flag later does not replay suppressed work; the next pull request event queues a fresh attempt.
+
Queued work and retries use the current personal setting.
Each successful automatic assignment records its owner and origin. A later reviewer edit, claim, or verified PR refresh can reassess that owner.
The worker replaces owners only when GitHub's current assignees exactly match its latest record; extra assignees, manual removal, and unknown-origin assignments remain authoritative.
It adds the replacement before removing the previous automatic owner and stops if GitHub reports a concurrent assignee change.
+Each GitHub write rechecks the delivery control, including owner removal and rollback after a partial reassignment.
If no report recipient qualifies, it clears only its recorded automatic assignment. A failed eligibility lookup leaves the assignment unchanged.
Incomplete human-edit history blocks automatic assignment.
The legacy DRI path also respects report exclusions and rechecks human choices before assignment.
diff --git a/products/signals/backend/reviewer_pr_assignment.py b/products/signals/backend/reviewer_pr_assignment.py
index b45a6905bb35..4f8446bd482f 100644
--- a/products/signals/backend/reviewer_pr_assignment.py
+++ b/products/signals/backend/reviewer_pr_assignment.py
@@ -16,7 +16,7 @@ still assigned, because a claim is a person's own decision.
from __future__ import annotations
import json
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, Literal
from uuid import UUID
from django.conf import settings
@@ -27,11 +27,12 @@ import structlog
from pydantic import ValidationError
from posthog.dataclasses import frozen
+from posthog.egress.github.transport import GitHubRateLimitError
from posthog.egress.limiter.policies import Priority
from posthog.event_usage import groups
from posthog.models import Team
from posthog.models.github_integration_base import PullRequestRef
-from posthog.models.integration import GitHubIntegration
+from posthog.models.integration import GitHubIntegration, GitHubIntegrationError
from posthog.ph_client import feature_enabled_or_false, ph_scoped_capture
from products.signals.backend.artefact_schemas import (
@@ -90,6 +91,67 @@ ASSIGNABLE_PR_STATES = frozenset(
}
)
+FirstAssignmentStatus = Literal["assigned", "skipped", "retryable_failure", "permanent_failure", "unknown_write"]
+FailureStatus = Literal["retryable_failure", "permanent_failure"]
+
+
+@frozen
+class FirstAssignment:
+ """One attempt to assign a PR owner. Unknown writes require a fresh GitHub read before retry."""
+
+ status: FirstAssignmentStatus
+ reason: str | None = None
+ assigned: tuple[str, ...] = ()
+ retry_after: int | None = None
+
+ @property
+ def retryable(self) -> bool:
+ return self.status in ("retryable_failure", "unknown_write")
+
+ @classmethod
+ def skipped(cls, reason: str) -> FirstAssignment:
+ return cls(status="skipped", reason=reason)
+
+ @classmethod
+ def failed(cls, status: FailureStatus, reason: str, *, retry_after: int | None = None) -> FirstAssignment:
+ return cls(status=status, reason=reason, retry_after=retry_after)
+
+
+def _failure_status(result: dict[str, Any]) -> FailureStatus:
+ """A GitHub client result without a status code never reached GitHub, so it is retryable.
+
+ A 429 or 5xx response is GitHub asking for a later attempt. Any other response GitHub sent is a
+ decision about this request, and sending it again cannot change it.
+ """
+ status_code = result.get("status_code")
+ if status_code is None or status_code == 429 or status_code >= 500:
+ return "retryable_failure"
+ return "permanent_failure"
+
+
+@frozen
+class PullRequestRead:
+ pr: dict[str, Any] | None = None
+ failure: FailureStatus | None = None
+ retry_after: int | None = None
+
+
+@frozen
+class AssigneeWrite:
+ """What GitHub reports after an add-assignees call. `unknown` means the call may have landed."""
+
+ assignees: tuple[str, ...] = ()
+ failure: FailureStatus | None = None
+ retry_after: int | None = None
+ unknown: bool = False
+
+
+@frozen
+class UnassignmentRead:
+ unassigned: bool | None = None
+ failure: FailureStatus | None = None
+ retry_after: int | None = None
+
def schedule_reviewer_pr_assignment(
*,
@@ -274,33 +336,19 @@ def ranked_reviewer_logins(*, team_id: int, report_id: str) -> list[str]:
return list(dict.fromkeys(login for login in candidates if login))
-def _github_for_repository(*, team_id: int, report_id: str, repository: str) -> GitHubIntegration | None:
- """The team's GitHub integration for a repository, or None when there is none to call."""
- try:
- return GitHubIntegration.first_for_team_repository(
- team_id, repository, source="signals_reviewers", priority=Priority.BATCH
- )
- except Exception:
- logger.exception(
- "signals.reviewer_pr_assignment.integration_lookup_failed",
- team_id=team_id,
- report_id=report_id,
- repository=repository,
- )
- return None
-
-
def _assignable_pull_request(
github: GitHubIntegration, *, team_id: int, report_id: str, parsed: PullRequestRef
-) -> dict[str, Any] | None:
- """The pull request as GitHub reports it, or None when it no longer accepts assignees.
+) -> PullRequestRead:
+ """The pull request as GitHub reports it, with no `pr` when it no longer accepts assignees.
A pull request closed or merged since the assignment was queued must not be reopened in
somebody's assigned list, so this reads the state from GitHub rather than trusting the queued
- snapshot. A read that fails counts as not assignable.
+ snapshot. A read that fails counts as not assignable and carries whether a retry can help.
"""
try:
pr = github.get_pull_request(parsed.repository, parsed.number)
+ except GitHubRateLimitError as error:
+ return PullRequestRead(failure="retryable_failure", retry_after=error.retry_after)
except Exception:
logger.exception(
"signals.reviewer_pr_assignment.pr_fetch_failed",
@@ -309,7 +357,7 @@ def _assignable_pull_request(
repository=parsed.repository,
pr_number=parsed.number,
)
- return None
+ return PullRequestRead(failure="retryable_failure")
if not pr.get("success"):
logger.warning(
"signals.reviewer_pr_assignment.pr_fetch_failed",
@@ -319,39 +367,43 @@ def _assignable_pull_request(
pr_number=parsed.number,
error=pr.get("error"),
)
- return None
+ return PullRequestRead(failure=_failure_status(pr))
if pr.get("merged") or pr.get("state") == "closed":
- return None
- return pr
+ return PullRequestRead()
+ return PullRequestRead(pr=pr)
def _add_assignees(
github: GitHubIntegration, *, team_id: int, report_id: str, parsed: PullRequestRef, logins: list[str]
-) -> list[str] | None:
+) -> AssigneeWrite:
"""Add the logins to the pull request.
- Returns the pull request's assignees as GitHub reports them after the call, an empty list when
- GitHub refused the call, and None when the outcome is unknown. A request that failed in transit
- may still have reached GitHub, so the caller must not assign anybody else on top of it.
+ Returns the pull request's assignees as GitHub reports them after the call, a failure when
+ GitHub refused the call, and `unknown` when the outcome is unknown. A request that failed in
+ transit may still have reached GitHub, so the caller must not assign anybody else on top of it.
"""
log = logger.bind(team_id=team_id, report_id=report_id, repository=parsed.repository, pr_number=parsed.number)
try:
if not assignment_side_effects_allowed(team_id=team_id, report_id=report_id):
- return None
+ return AssigneeWrite(failure="permanent_failure")
result = github.add_pull_request_assignees(parsed.repository, parsed.number, logins)
+ except GitHubRateLimitError as error:
+ return AssigneeWrite(failure="retryable_failure", retry_after=error.retry_after)
except Exception:
log.exception("signals.reviewer_pr_assignment.assign_failed")
- return None
+ return AssigneeWrite(unknown=True)
if not result.get("success"):
log.warning("signals.reviewer_pr_assignment.assign_failed", error=result.get("error"))
# The client reports a status code only for a response GitHub sent.
- return [] if result.get("status_code") is not None else None
+ if result.get("status_code") is None:
+ return AssigneeWrite(unknown=True)
+ return AssigneeWrite(failure=_failure_status(result))
- assigned = list(result.get("assignees") or [])
+ assigned = tuple(result.get("assignees") or ())
# Counts only: GitHub logins are member PII and must not reach logs. A `requested` above
# `assigned` means GitHub dropped a login, usually one without push access to the repository.
log.info("signals.reviewer_pr_assignment.assigned", requested=len(logins), assigned=len(assigned))
- return assigned
+ return AssigneeWrite(assignees=assigned)
def _request_reviews(
@@ -435,6 +487,15 @@ class OwnerApplicationState:
allowed: bool
previous: PROwnerAssignment | None
reason: str
+ failure: FailureStatus | None = None
+ retry_after: int | None = None
+
+
+@frozen
+class OwnerCandidates:
+ owners: tuple[AssignableOwner, ...] = ()
+ failure: FailureStatus | None = None
+ retry_after: int | None = None
def _assignable_owners(
@@ -447,7 +508,7 @@ def _assignable_owners(
assignments: PROwnerAssignmentService,
limit: int | None = None,
first_only: bool = False,
-) -> list[AssignableOwner] | None:
+) -> OwnerCandidates:
"""Candidates whose repository access and linked identity GitHub verifies."""
log = logger.bind(team_id=team_id, report_id=report_id, repository=parsed.repository, pr_number=parsed.number)
bounded_candidates = candidates[:limit] if limit is not None else candidates
@@ -457,18 +518,20 @@ def _assignable_owners(
identity = identities[login]
try:
result = github.is_assignable(parsed.repository, login)
+ except GitHubRateLimitError as error:
+ return OwnerCandidates(failure="retryable_failure", retry_after=error.retry_after)
except Exception:
log.exception("signals.reviewer_pr_assignment.assignable_check_failed")
- return None
+ return OwnerCandidates(failure="retryable_failure")
if not result.get("success"):
log.warning("signals.reviewer_pr_assignment.assignable_check_failed", error=result.get("error"))
- return None
+ return OwnerCandidates(failure=_failure_status(result))
if not result.get("assignable"):
continue
if not identity.linked_user:
owners.append(AssignableOwner(login=login, identity=None))
if first_only:
- return owners
+ return OwnerCandidates(owners=tuple(owners))
continue
if identity.proof is None:
log.info("signals.pr_owner.identity_unverified")
@@ -479,14 +542,19 @@ def _assignable_owners(
login,
expected_user_id=identity.proof.github_user_id,
)
+ except GitHubRateLimitError as error:
+ return OwnerCandidates(failure="retryable_failure", retry_after=error.retry_after)
+ except GitHubIntegrationError as error:
+ log.warning("signals.pr_owner.identity_check_failed", status_code=error.status_code)
+ return OwnerCandidates(failure=_failure_status({"status_code": error.status_code}))
except Exception:
log.exception("signals.pr_owner.identity_check_failed")
- return None
+ return OwnerCandidates(failure="retryable_failure")
if permission in {"admin", "write"}:
owners.append(AssignableOwner(login=login, identity=identity.proof))
if first_only:
- return owners
- return owners
+ return OwnerCandidates(owners=tuple(owners))
+ return OwnerCandidates(owners=tuple(owners))
def _owner_application_state(
@@ -508,7 +576,16 @@ def _owner_application_state(
return OwnerApplicationState(allowed=False, previous=previous, reason=reason)
if current:
return OwnerApplicationState(allowed=False, previous=None, reason="unknown_assignment_origin")
- if _was_unassigned_by_hand(github, team_id=team_id, report_id=report_id, parsed=parsed):
+ history = _was_unassigned_by_hand(github, team_id=team_id, report_id=report_id, parsed=parsed)
+ if history.failure:
+ return OwnerApplicationState(
+ allowed=False,
+ previous=None,
+ reason="events_fetch_failed",
+ failure=history.failure,
+ retry_after=history.retry_after,
+ )
+ if history.unassigned:
return OwnerApplicationState(allowed=False, previous=None, reason="manually_unassigned")
return OwnerApplicationState(allowed=True, previous=None, reason="unassigned")
@@ -554,6 +631,8 @@ def _remove_assignees(
logins: list[str],
) -> list[str] | None:
try:
+ if not assignment_side_effects_allowed(team_id=team_id, report_id=report_id):
+ return None
result = github.remove_pull_request_assignees(parsed.repository, parsed.number, logins)
except Exception:
logger.exception("signals.pr_owner.unassign_failed", team_id=team_id, report_id=report_id)
@@ -579,7 +658,7 @@ def _apply_automatic_owners(
application: OwnerApplicationState,
owners: list[AssignableOwner],
origin: PROwnerOrigin,
-) -> list[str]:
+) -> FirstAssignment:
previous = application.previous
previous_logins = [owner.login for owner in previous.owners] if previous else []
wanted = [owner.login for owner in owners]
@@ -603,14 +682,22 @@ def _apply_automatic_owners(
origin=origin,
verified_count=verified_count,
)
- return []
+ return FirstAssignment.skipped("same_candidates" if previous else "no_assignable_recipients")
added = [login for login in wanted if login not in previous_set]
if added:
- assigned = _add_assignees(github, team_id=team_id, report_id=report_id, parsed=parsed, logins=added)
+ write = _add_assignees(github, team_id=team_id, report_id=report_id, parsed=parsed, logins=added)
+ if write.unknown or write.failure:
+ if previous is not None:
+ return FirstAssignment.skipped("reassignment_failed")
+ if write.unknown:
+ return FirstAssignment(status="unknown_write", reason="assign_failed")
+ if write.failure:
+ return FirstAssignment.failed(write.failure, "assign_failed", retry_after=write.retry_after)
+ assigned = list(write.assignees)
expected = previous_set | set(added)
- if assigned is None or {login.lower() for login in assigned} != expected:
- if assigned is not None:
+ if {login.lower() for login in assigned} != expected:
+ if assigned:
_remove_assignees(github, team_id=team_id, report_id=report_id, parsed=parsed, logins=added)
if previous is not None:
_capture_owner_reassessment(
@@ -622,7 +709,7 @@ def _apply_automatic_owners(
origin=origin,
verified_count=verified_count,
)
- return []
+ return FirstAssignment.skipped("assignees_changed_during_reassessment")
removed = [login for login in previous_logins if login not in wanted_set]
if removed:
@@ -640,7 +727,7 @@ def _apply_automatic_owners(
origin=origin,
verified_count=verified_count,
)
- return []
+ return FirstAssignment.skipped("assignees_changed_during_reassessment")
assignments.record(identities=identities, origin=origin, replaced_logins=previous_logins)
if previous is not None:
@@ -653,7 +740,7 @@ def _apply_automatic_owners(
origin=origin,
verified_count=verified_count,
)
- return added
+ return FirstAssignment(status="assigned" if added else "skipped", assigned=tuple(added))
def _owning_team(
@@ -672,16 +759,17 @@ def _owning_team(
return None
-def _was_unassigned_by_hand(github: GitHubIntegration, *, team_id: int, report_id: str, parsed: PullRequestRef) -> bool:
- """Whether somebody removed an assignee from the pull request. A failed read counts as yes.
-
- A person who unassigned the DRI made a decision, and a later report event must not undo it.
- """
+def _was_unassigned_by_hand(
+ github: GitHubIntegration, *, team_id: int, report_id: str, parsed: PullRequestRef
+) -> UnassignmentRead:
+ """Read manual removals. A failed read must not allow an assignment."""
try:
result = github.was_ever_unassigned(parsed.repository, parsed.number)
+ except GitHubRateLimitError as error:
+ return UnassignmentRead(failure="retryable_failure", retry_after=error.retry_after)
except Exception:
logger.exception("signals.reviewer_pr_assignment.events_fetch_failed", team_id=team_id, report_id=report_id)
- return True
+ return UnassignmentRead(failure="retryable_failure")
if not result.get("success"):
logger.warning(
"signals.reviewer_pr_assignment.events_fetch_failed",
@@ -689,8 +777,8 @@ def _was_unassigned_by_hand(github: GitHubIntegration, *, team_id: int, report_i
report_id=report_id,
error=result.get("error"),
)
- return True
- return bool(result.get("unassigned"))
+ return UnassignmentRead(failure=_failure_status(result))
+ return UnassignmentRead(unassigned=bool(result.get("unassigned")))
def dri_candidate_logins(
@@ -760,7 +848,7 @@ def _assign_report_reviewers(
parsed: PullRequestRef,
pr: dict[str, Any],
standing: StandingExclusions,
-) -> list[str]:
+) -> FirstAssignment:
assignments = PROwnerAssignmentService(
team_id=team_id,
report_id=report_id,
@@ -775,6 +863,8 @@ def _assign_report_reviewers(
pr=pr,
assignments=assignments,
)
+ if application.failure:
+ return FirstAssignment.failed(application.failure, application.reason, retry_after=application.retry_after)
if not application.allowed:
if application.previous is not None:
_capture_owner_reassessment(
@@ -784,14 +874,14 @@ def _assign_report_reviewers(
outcome="protected",
reason=application.reason,
)
- return []
+ return FirstAssignment.skipped(application.reason)
log = logger.bind(team_id=team_id, report_id=report_id)
try:
team = Team.objects.get(id=team_id)
snapshot = ReviewerRules.read_for_report(team_id=team_id, report_id=report_id)
if snapshot.corrections.needs_review:
- return []
+ return FirstAssignment.skipped("needs_review")
candidates = standing.allowed_logins(_report_assignee_logins(team, snapshot))
if len(candidates) < len(snapshot.reviewers):
log.info(
@@ -799,7 +889,7 @@ def _assign_report_reviewers(
selected=len(snapshot.reviewers),
linked=len(candidates),
)
- owners = _assignable_owners(
+ candidates_read = _assignable_owners(
github,
team_id=team_id,
report_id=report_id,
@@ -809,35 +899,46 @@ def _assign_report_reviewers(
limit=MAX_REPORT_ASSIGNEE_CHECKS,
first_only=True,
)
- if owners is None:
- return []
+ if candidates_read.failure:
+ if application.previous is not None:
+ return FirstAssignment.skipped("reassessment_lookup_failed")
+ return FirstAssignment.failed(
+ candidates_read.failure, "assignable_check_failed", retry_after=candidates_read.retry_after
+ )
+ owners = list(candidates_read.owners)
current_pr = _assignable_pull_request(github, team_id=team_id, report_id=report_id, parsed=parsed)
- if current_pr is None:
- return []
+ if current_pr.failure and application.previous is None:
+ return FirstAssignment.failed(current_pr.failure, "pr_fetch_failed", retry_after=current_pr.retry_after)
+ if current_pr.pr is None:
+ return FirstAssignment.skipped("pull_request_changed")
current_application = _owner_application_state(
github,
team_id=team_id,
report_id=report_id,
parsed=parsed,
- pr=current_pr,
+ pr=current_pr.pr,
assignments=assignments,
)
+ if current_application.failure:
+ return FirstAssignment.failed(
+ current_application.failure, current_application.reason, retry_after=current_application.retry_after
+ )
if not current_application.allowed or current_application.previous != application.previous:
- return []
+ return FirstAssignment.skipped("pull_request_changed")
current = ReviewerRules.read_for_report(team_id=team_id, report_id=report_id)
# An instruction saved during the GitHub calls must still stop the assignment.
if (
current.state != snapshot.state
or standing.refreshed().allowed_logins(_report_assignee_logins(team, current)) != candidates
):
- return []
+ return FirstAssignment.skipped("report_changed")
if not (
assignment_writer_enabled(team_id)
and team_research_reviewer_selection_enabled(team_id)
and assignment_side_effects_enabled(team_id)
):
- return []
+ return FirstAssignment.skipped("delivery_disabled")
return _apply_automatic_owners(
github,
team_id=team_id,
@@ -848,9 +949,15 @@ def _assign_report_reviewers(
owners=owners,
origin="report",
)
+ except GitHubRateLimitError as error:
+ if application.previous is not None:
+ return FirstAssignment.skipped("reassessment_lookup_failed")
+ return FirstAssignment.failed("retryable_failure", "rate_limited", retry_after=error.retry_after)
except Exception:
log.exception("signals.reviewer_pr_assignment.report_assignment_failed")
- return []
+ if application.previous is not None:
+ return FirstAssignment.skipped("reassessment_failed")
+ return FirstAssignment.failed("retryable_failure", "error")
def assign_reviewers_to_pull_request(*, team_id: int, report_id: str, pr_url: str) -> list[str]:
@@ -859,40 +966,95 @@ def assign_reviewers_to_pull_request(*, team_id: int, report_id: str, pr_url: st
Returns newly added assignees. An empty result can also mean a recorded automatic owner
was cleared. Manual assignments remain authoritative.
"""
+ return list(deliver_pull_request_assignment(team_id=team_id, report_id=report_id, pr_url=pr_url).assigned)
+
+
+def deliver_pull_request_assignment(*, team_id: int, report_id: str, pr_url: str) -> FirstAssignment:
+ """One delivery attempt, with the outcome the worker uses to decide on a bounded retry.
+
+ Only the shared report assignment reports a retryable outcome. The legacy DRI and opt-in
+ policies stay best effort and report `assigned` or `skipped`.
+ """
if not assignment_side_effects_allowed(team_id=team_id, report_id=report_id):
- return []
+ return FirstAssignment.skipped("delivery_disabled")
if not SignalReport.objects.filter(id=report_id, team_id=team_id).exists():
- return []
+ return FirstAssignment.skipped("report_missing")
shared_assignment = assignment_writer_enabled(team_id) and team_research_reviewer_selection_enabled(team_id)
if shared_assignment and not assignment_side_effects_enabled(team_id):
- return []
+ return FirstAssignment.skipped("delivery_disabled")
logins = [] if shared_assignment else opted_in_assignee_logins(team_id=team_id, report_id=report_id)
reviewers = ranked_reviewer_logins(team_id=team_id, report_id=report_id)
dri_enabled = _pr_dri_enabled(team_id)
if not reviewers and not logins and not dri_enabled and not shared_assignment:
- return []
+ return FirstAssignment.skipped("nothing_to_do")
parsed = GitHubIntegration.parse_pull_request_url(pr_url)
if parsed is None:
- return []
-
- github = _github_for_repository(team_id=team_id, report_id=report_id, repository=parsed.repository)
+ return FirstAssignment.failed("permanent_failure", "invalid_pr_url")
+
+ first_delivery = (
+ shared_assignment
+ and PROwnerAssignmentService(
+ team_id=team_id, report_id=report_id, repository=parsed.repository, pr_number=parsed.number
+ ).latest()
+ is None
+ )
+ try:
+ github = GitHubIntegration.first_for_team_repository(
+ team_id, parsed.repository, source="signals_reviewers", priority=Priority.BATCH
+ )
+ except GitHubRateLimitError as error:
+ return (
+ FirstAssignment.failed("retryable_failure", "integration_lookup_failed", retry_after=error.retry_after)
+ if first_delivery
+ else FirstAssignment.skipped("integration_lookup_failed")
+ )
+ except Exception:
+ logger.exception(
+ "signals.reviewer_pr_assignment.integration_lookup_failed",
+ team_id=team_id,
+ report_id=report_id,
+ repository=parsed.repository,
+ )
+ return (
+ FirstAssignment.failed("retryable_failure", "integration_lookup_failed")
+ if first_delivery
+ else FirstAssignment.skipped("integration_lookup_failed")
+ )
if github is None:
- return []
-
- pr = _assignable_pull_request(github, team_id=team_id, report_id=report_id, parsed=parsed)
- if pr is None:
- return []
+ return FirstAssignment.failed("permanent_failure", "no_integration")
+
+ read = _assignable_pull_request(github, team_id=team_id, report_id=report_id, parsed=parsed)
+ if read.failure:
+ return (
+ FirstAssignment.failed(read.failure, "pr_fetch_failed", retry_after=read.retry_after)
+ if first_delivery
+ else FirstAssignment.skipped("pr_fetch_failed")
+ )
+ if read.pr is None:
+ return FirstAssignment.skipped("pull_request_closed")
+ pr = read.pr
standing = StandingExclusions.load(team_id=team_id, report_id=report_id, repository=parsed.repository)
logins = standing.allowed_logins(logins)
reviewers = standing.allowed_logins(reviewers)
if shared_assignment:
- assigned = _assign_report_reviewers(
+ first = _assign_report_reviewers(
github, team_id=team_id, report_id=report_id, parsed=parsed, pr=pr, standing=standing
)
+ logger.info(
+ "signals.reviewer_pr_assignment.first_assignment",
+ team_id=team_id,
+ report_id=report_id,
+ status=first.status,
+ reason=first.reason,
+ assigned=len(first.assigned),
+ )
+ if first.status not in ("assigned", "skipped"):
+ return first
+ assigned = list(first.assigned)
elif dri_enabled:
assigned = _assign_one_dri(
github,
@@ -904,12 +1066,14 @@ def assign_reviewers_to_pull_request(*, team_id: int, report_id: str, pr_url: st
reviewers=reviewers,
standing=standing,
)
+ first = FirstAssignment(status="assigned" if assigned else "skipped", assigned=tuple(assigned))
else:
assigned = (
- _add_assignees(github, team_id=team_id, report_id=report_id, parsed=parsed, logins=logins) or []
+ list(_add_assignees(github, team_id=team_id, report_id=report_id, parsed=parsed, logins=logins).assignees)
if logins
else []
)
+ first = FirstAssignment(status="assigned" if assigned else "skipped", assigned=tuple(assigned))
if shared_assignment:
if not (
@@ -917,21 +1081,21 @@ def assign_reviewers_to_pull_request(*, team_id: int, report_id: str, pr_url: st
and team_research_reviewer_selection_enabled(team_id)
and assignment_side_effects_enabled(team_id)
):
- return assigned
+ return first
try:
snapshot = ReviewerRules.read_for_report(team_id=team_id, report_id=report_id)
if snapshot.corrections.needs_review:
- return assigned
+ return first
reviewers = _report_assignee_logins(Team.objects.get(id=team_id), snapshot, require_opt_in=False)
except Exception:
logger.exception(
"signals.reviewer_pr_assignment.reviewers_lookup_failed", team_id=team_id, report_id=report_id
)
- return assigned
- current_pr = _assignable_pull_request(github, team_id=team_id, report_id=report_id, parsed=parsed)
- if current_pr is None:
- return assigned
- pr = current_pr
+ return first
+ current = _assignable_pull_request(github, team_id=team_id, report_id=report_id, parsed=parsed)
+ if current.pr is None:
+ return first
+ pr = current.pr
# An instruction saved during the assignment calls must still stop the review request.
_request_reviews(
github,
@@ -942,7 +1106,7 @@ def assign_reviewers_to_pull_request(*, team_id: int, report_id: str, pr_url: st
reviewers=reviewers,
standing=standing.refreshed(),
)
- return assigned
+ return first
def _assign_one_dri(
@@ -1005,7 +1169,7 @@ def _assign_one_dri(
),
)
]
- owners = _assignable_owners(
+ candidates_read = _assignable_owners(
github,
team_id=team_id,
report_id=report_id,
@@ -1015,18 +1179,18 @@ def _assign_one_dri(
limit=MAX_DRI_CHECKS,
first_only=True,
)
- if not owners:
+ if not candidates_read.owners:
return []
- owner = owners[0]
+ owner = candidates_read.owners[0]
current_pr = _assignable_pull_request(github, team_id=team_id, report_id=report_id, parsed=parsed)
- if current_pr is None:
+ if current_pr.pr is None:
return []
current_application = _owner_application_state(
github,
team_id=team_id,
report_id=report_id,
parsed=parsed,
- pr=current_pr,
+ pr=current_pr.pr,
assignments=assignments,
)
if not current_application.allowed or current_application.previous != application.previous:
@@ -1041,7 +1205,7 @@ def _assign_one_dri(
or not (owner.login == claimant or fresh.allowed_logins([owner.login]))
):
return []
- return _apply_automatic_owners(
+ outcome = _apply_automatic_owners(
github,
team_id=team_id,
report_id=report_id,
@@ -1051,3 +1215,5 @@ def _assign_one_dri(
owners=[owner],
origin="claim" if claimant == owner.login else "legacy",
)
+
+ return list(outcome.assigned)
diff --git a/products/signals/backend/tasks.py b/products/signals/backend/tasks.py
index 240174d8b03a..8940f383b3b6 100644
--- a/products/signals/backend/tasks.py
+++ b/products/signals/backend/tasks.py
@@ -60,7 +60,7 @@ from products.signals.backend.report_generation.repo_activity import (
rebuild_repository_activity,
repository_activity_needs_rebuild,
)
-from products.signals.backend.reviewer_pr_assignment import assign_reviewers_to_pull_request
+from products.signals.backend.reviewer_pr_assignment import deliver_pull_request_assignment
from products.signals.backend.reviewer_pr_ready import open_pull_request_ready_for_review
from products.signals.backend.scout_harness.background_bands import refresh_background_bands
from products.signals.backend.scout_harness.inactivity import sweep_inactive_scouts
@@ -733,20 +733,43 @@ def send_reviewer_added_slack_notifications(
)
+# Bounded recovery for a transient GitHub failure on the first shared assignment: 1m, 2m, 4m, 8m,
+# 15m, about half an hour in total. A later pull request event still queues a fresh attempt, so an
+# exhausted retry is final only for this attempt and never sweeps other reports.
+_FIRST_ASSIGNMENT_MAX_RETRIES = 5
+_FIRST_ASSIGNMENT_RETRY_BASE_SECONDS = 60
+_FIRST_ASSIGNMENT_RETRY_MAX_SECONDS = 900
+
+
@shared_task(
name="products.signals.backend.tasks.assign_reviewers_on_implementation_pr",
ignore_result=True,
- max_retries=0,
+ bind=True,
+ max_retries=_FIRST_ASSIGNMENT_MAX_RETRIES,
)
@with_team_scope()
-def assign_reviewers_on_implementation_pr(team_id: int, report_id: str, pr_url: str) -> None:
- """Put a report's opted-in reviewers, or else one DRI, on its implementation PR as GitHub assignees.
+def assign_reviewers_on_implementation_pr(self, team_id: int, report_id: str, pr_url: str) -> None:
+ """Assign an owner to the implementation PR and request a review.
Runs on a worker because the GitHub calls (integration probe, PR read, assign) must not hold up
- the claim, sync, or reviewer edit that queued it. Best-effort end to end, so the assigner
- reports its own failures and this never retries: the next pull request event queues it again.
+ the claim, sync, or reviewer edit that queued it. Every attempt re-reads the accepted list,
+ corrections, access, delivery flags, and GitHub state, so a retry respects anything that changed
+ in between and finds a write that did land as an existing assignment. A skipped attempt, such as
+ one a disabled flag suppressed, is a decision and is never retried or replayed.
"""
- assign_reviewers_to_pull_request(team_id=team_id, report_id=report_id, pr_url=pr_url)
+ outcome = deliver_pull_request_assignment(team_id=team_id, report_id=report_id, pr_url=pr_url)
+ if not outcome.retryable:
+ return
+ log = logger.bind(team_id=team_id, report_id=report_id, status=outcome.status, reason=outcome.reason)
+ if self.request.retries >= self.max_retries:
+ log.warning("signals.reviewer_pr_assignment.recovery_exhausted", attempts=self.request.retries + 1)
+ return
+ countdown = min(
+ _FIRST_ASSIGNMENT_RETRY_BASE_SECONDS * (2**self.request.retries), _FIRST_ASSIGNMENT_RETRY_MAX_SECONDS
+ )
+ countdown = max(countdown, outcome.retry_after or 0)
+ log.info("signals.reviewer_pr_assignment.recovery_scheduled", attempt=self.request.retries + 1, countdown=countdown)
+ raise self.retry(countdown=countdown)
@shared_task(
diff --git a/products/signals/backend/test/test_report_pr_assignment.py b/products/signals/backend/test/test_report_pr_assignment.py
index c98827a8c7e5..59887e0c524e 100644
--- a/products/signals/backend/test/test_report_pr_assignment.py
+++ b/products/signals/backend/test/test_report_pr_assignment.py
@@ -3,8 +3,12 @@ import json
import pytest
from unittest.mock import MagicMock, patch
+from celery.exceptions import Retry
+
from posthog.constants import AvailableFeature
+from posthog.egress.github.transport import GitHubRateLimitError
from posthog.models import Organization, Team
+from posthog.models.integration import GitHubIntegrationError
from posthog.models.organization import OrganizationMembership
from products.access_control.backend.models.access_control import AccessControl
@@ -18,7 +22,12 @@ from products.signals.backend.artefact_schemas import (
from products.signals.backend.models import SignalReport, SignalReportArtefact, SignalUserAutonomyConfig
from products.signals.backend.report_assignments import claim_report
from products.signals.backend.reviewer_exclusions import create_exclusion
-from products.signals.backend.reviewer_pr_assignment import assign_reviewers_to_pull_request
+from products.signals.backend.reviewer_pr_assignment import (
+ FirstAssignment,
+ assign_reviewers_to_pull_request,
+ deliver_pull_request_assignment,
+)
+from products.signals.backend.tasks import assign_reviewers_on_implementation_pr
from products.signals.backend.test.test_reviewer_pr_assignment import (
_NOT_UNASSIGNED,
PR_URL,
@@ -82,6 +91,13 @@ class TestReportPRAssignment:
):
return assign_reviewers_to_pull_request(team_id=report.team_id, report_id=str(report.id), pr_url=PR_URL)
+ def deliver(self, report: SignalReport, github: MagicMock) -> FirstAssignment:
+ with patch(
+ "products.signals.backend.reviewer_pr_assignment.GitHubIntegration.first_for_team_repository",
+ return_value=github,
+ ):
+ return deliver_pull_request_assignment(team_id=report.team_id, report_id=str(report.id), pr_url=PR_URL)
+
@pytest.mark.parametrize("dri_enabled", [False, True])
@pytest.mark.parametrize("first_assignable", [False, True])
def test_pr_selects_one_eligible_owner_without_changing_report_recipients(
@@ -333,3 +349,305 @@ class TestReportPRAssignment:
expected_user_id=int(selected.social_auth.get(provider="github").uid),
)
github.add_pull_request_assignees.assert_not_called()
+
+ @pytest.mark.parametrize(
+ "failure,expected",
+ [
+ ("pr-network", "retryable_failure"),
+ ("pr-not-found", "permanent_failure"),
+ ("pr-closed", "skipped"),
+ ("events-network", "retryable_failure"),
+ ("events-forbidden", "permanent_failure"),
+ ("events-forbidden-second-read", "permanent_failure"),
+ ("check-network", "retryable_failure"),
+ ("check-forbidden", "permanent_failure"),
+ ("permission-403", "permanent_failure"),
+ ("permission-429", "retryable_failure"),
+ ("permission-503", "retryable_failure"),
+ ("permission-network", "retryable_failure"),
+ ("write-raises", "unknown_write"),
+ ("write-bad-gateway", "retryable_failure"),
+ ("write-unprocessable", "permanent_failure"),
+ ("delivery-off", "skipped"),
+ ],
+ )
+ def test_only_transient_and_unknown_delivery_failures_are_retryable(self, org_and_team, failure, expected):
+ org, team = org_and_team
+ _make_reviewer(org, "selected", opted_in=True)
+ report = _make_report(team, ["selected"])
+ github = self.github()
+ if failure.startswith("pr-"):
+ github.get_pull_request.side_effect = None
+ if failure == "pr-network":
+ github.get_pull_request.return_value = {"success": False, "error": "network"}
+ elif failure == "pr-not-found":
+ github.get_pull_request.return_value = {"success": False, "error": "gone", "status_code": 404}
+ elif failure == "pr-closed":
+ github.get_pull_request.return_value["state"] = "closed"
+ elif failure == "events-network":
+ github.was_ever_unassigned.return_value = {"success": False, "error": "network"}
+ elif failure.startswith("events-forbidden"):
+ github.was_ever_unassigned.side_effect = ([_NOT_UNASSIGNED] if failure.endswith("second-read") else []) + [
+ {"success": False, "status_code": 403}
+ ]
+ elif failure == "check-network":
+ github.is_assignable.return_value = {"success": False, "error": "network"}
+ elif failure == "check-forbidden":
+ github.is_assignable.return_value = {"success": False, "error": "forbidden", "status_code": 403}
+ elif failure.startswith("permission-"):
+ status_code = None if failure == "permission-network" else int(failure.removeprefix("permission-"))
+ github.get_collaborator_permission.side_effect = GitHubIntegrationError(
+ "Permission lookup failed", status_code=status_code
+ )
+ elif failure == "write-raises":
+ github.add_pull_request_assignees.side_effect = RuntimeError("socket closed")
+ elif failure == "write-bad-gateway":
+ github.add_pull_request_assignees.side_effect = None
+ github.add_pull_request_assignees.return_value = {"success": False, "error": "bad", "status_code": 502}
+ elif failure == "write-unprocessable":
+ github.add_pull_request_assignees.side_effect = None
+ github.add_pull_request_assignees.return_value = {"success": False, "error": "bad", "status_code": 422}
+
+ if failure == "delivery-off":
+ with patch(
+ "products.signals.backend.reviewer_pr_assignment.assignment_side_effects_enabled", return_value=False
+ ):
+ outcome = self.deliver(report, github)
+ else:
+ outcome = self.deliver(report, github)
+
+ assert outcome.status == expected
+ assert outcome.retryable is (expected in {"retryable_failure", "unknown_write"})
+ assert outcome.assigned == ()
+ if failure.startswith("permission-"):
+ github.add_pull_request_assignees.assert_not_called()
+ github.request_pull_request_reviews.assert_not_called()
+
+ @pytest.mark.parametrize(
+ "step",
+ ["integration", "pr", "pr-second-read", "history", "history-second-read", "assignability", "identity", "write"],
+ )
+ def test_rate_limit_delay_reaches_the_worker(self, org_and_team: tuple[Organization, Team], step: str) -> None:
+ org, team = org_and_team
+ _make_reviewer(org, "selected", opted_in=True)
+ report = _make_report(team, ["selected"])
+ github = self.github()
+ error = GitHubRateLimitError("rate limited", retry_after=3600)
+ if step.startswith("pr"):
+ github.get_pull_request.side_effect = (
+ [github.get_pull_request()] if step.endswith("second-read") else []
+ ) + [error]
+ elif step.startswith("history"):
+ github.was_ever_unassigned.side_effect = ([_NOT_UNASSIGNED] if step.endswith("second-read") else []) + [
+ error
+ ]
+ elif step == "assignability":
+ github.is_assignable.side_effect = error
+ elif step == "identity":
+ github.get_collaborator_permission.side_effect = error
+ elif step == "write":
+ github.add_pull_request_assignees.side_effect = error
+
+ with patch(
+ "products.signals.backend.reviewer_pr_assignment.GitHubIntegration.first_for_team_repository",
+ return_value=github,
+ side_effect=error if step == "integration" else None,
+ ):
+ outcome = deliver_pull_request_assignment(team_id=team.id, report_id=str(report.id), pr_url=PR_URL)
+
+ assert outcome.retryable
+ assert outcome.retry_after == 3600
+ assert github.add_pull_request_assignees.call_count == (1 if step == "write" else 0)
+ github.request_pull_request_reviews.assert_not_called()
+
+ @pytest.mark.parametrize("dri_enabled", [False, True])
+ @pytest.mark.parametrize(
+ "failure", ["integration-network", "integration-limit", "pr-network", "pr-limit", "pr-503"]
+ )
+ def test_legacy_failures_do_not_retry(
+ self, org_and_team: tuple[Organization, Team], dri_flag: MagicMock, dri_enabled: bool, failure: str
+ ) -> None:
+ org, team = org_and_team
+ dri_flag.return_value = dri_enabled
+ _make_reviewer(org, "selected", opted_in=True)
+ report = _make_report(team, ["selected"])
+ github = self.github()
+ error = (
+ GitHubRateLimitError("rate limited", retry_after=3600) if failure.endswith("limit") else OSError("offline")
+ )
+ if failure == "pr-503":
+ github.get_pull_request.side_effect = None
+ github.get_pull_request.return_value = {"success": False, "status_code": 503}
+ elif failure.startswith("pr"):
+ github.get_pull_request.side_effect = error
+
+ with (
+ patch("products.signals.backend.reviewer_pr_assignment.assignment_writer_enabled", return_value=False),
+ patch(
+ "products.signals.backend.reviewer_pr_assignment.GitHubIntegration.first_for_team_repository",
+ return_value=github,
+ side_effect=error if failure.startswith("integration") else None,
+ ),
+ ):
+ outcome = deliver_pull_request_assignment(team_id=team.id, report_id=str(report.id), pr_url=PR_URL)
+
+ assert outcome.status == "skipped"
+ assert not outcome.retryable
+ github.add_pull_request_assignees.assert_not_called()
+
+ @pytest.mark.parametrize("landed,opted_in", [(True, True), (False, True), (False, False)])
+ def test_a_retry_after_an_unknown_write_reads_github_before_writing_again(
+ self, org_and_team: tuple[Organization, Team], landed: bool, opted_in: bool
+ ) -> None:
+ org, team = org_and_team
+ selected = _make_reviewer(org, "selected", opted_in=True)
+ report = _make_report(team, ["selected"])
+ github = self.github()
+ add_assignees = github.add_pull_request_assignees.side_effect
+ github.add_pull_request_assignees.side_effect = RuntimeError("socket closed")
+
+ assert self.deliver(report, github).status == "unknown_write"
+
+ github.add_pull_request_assignees.side_effect = add_assignees
+ if landed:
+ github.assignee_state[:] = ["selected"]
+ SignalUserAutonomyConfig.objects.filter(user=selected).update(github_assign_on_pull_request=opted_in)
+ retry = self.deliver(report, github)
+
+ if landed:
+ assert (retry.status, retry.reason) == ("skipped", "unknown_assignment_origin")
+ assert github.add_pull_request_assignees.call_count == 1
+ github.request_pull_request_reviews.assert_called_once_with("PostHog/posthog", 123, ["selected"])
+ elif not opted_in:
+ assert (retry.status, retry.reason) == ("skipped", "no_assignable_recipients")
+ assert not retry.retryable
+ assert github.add_pull_request_assignees.call_count == 1
+ else:
+ assert (retry.status, retry.assigned) == ("assigned", ("selected",))
+ assert github.add_pull_request_assignees.call_count == 2
+ assignment = PROwnerAssignment.model_validate_json(report.artefacts.get(type="pr_owner_assignment").content)
+ assert assignment.owners[0].login == "selected"
+ assert assignment.owners[0].proof is not None
+
+ @pytest.mark.parametrize("selection_enabled", [False, True])
+ def test_disabling_delivery_stops_owner_removal(
+ self, org_and_team: tuple[Organization, Team], dri_flag: MagicMock, selection_enabled: bool
+ ) -> None:
+ org, team = org_and_team
+ dri_flag.return_value = True
+ _make_reviewer(org, "selected", opted_in=True)
+ _make_reviewer(org, "replacement", opted_in=True)
+ report = _make_report(team, ["selected"])
+ github = self.github()
+ with patch(
+ "products.signals.backend.reviewer_pr_assignment.team_research_reviewer_selection_enabled",
+ return_value=selection_enabled,
+ ):
+ assert self.assign(report, github) == ["selected"]
+ SignalReportArtefact.objects.create(
+ team=team,
+ report=report,
+ type="suggested_reviewers",
+ content=json.dumps([{"github_login": "replacement"}]),
+ )
+ github.request_pull_request_reviews.reset_mock()
+ add_assignees = github.add_pull_request_assignees.side_effect
+ with patch(
+ "products.signals.backend.reviewer_pr_assignment.assignment_side_effects_allowed", return_value=True
+ ) as delivery_allowed:
+
+ def add_then_disable(repository: str, number: int, logins: list[str]) -> dict[str, object]:
+ result = add_assignees(repository, number, logins)
+ delivery_allowed.return_value = False
+ return result
+
+ github.add_pull_request_assignees.side_effect = add_then_disable
+ outcome = self.deliver(report, github)
+
+ assert outcome.status == "skipped"
+ assert not outcome.retryable
+ assert github.assignee_state == ["selected", "replacement"]
+ github.remove_pull_request_assignees.assert_not_called()
+ github.request_pull_request_reviews.assert_not_called()
+ assert report.artefacts.filter(type="pr_owner_assignment").count() == 1
+
+ @pytest.mark.parametrize("step", ["integration", "pr", "assignability", "identity", "write"])
+ def test_reassessment_failures_do_not_start_first_assignment_retries(
+ self, org_and_team: tuple[Organization, Team], step: str
+ ) -> None:
+ org, team = org_and_team
+ _make_reviewer(org, "selected", opted_in=True)
+ _make_reviewer(org, "replacement", opted_in=True)
+ report = _make_report(team, ["selected"])
+ github = self.github()
+ assert self.assign(report, github) == ["selected"]
+ SignalReportArtefact.objects.create(
+ team=team, report=report, type="suggested_reviewers", content=json.dumps([{"github_login": "replacement"}])
+ )
+ error = GitHubRateLimitError("rate limited", retry_after=3600)
+ boundary = {
+ "pr": github.get_pull_request,
+ "assignability": github.is_assignable,
+ "identity": github.get_collaborator_permission,
+ "write": github.add_pull_request_assignees,
+ }.get(step)
+ if boundary is not None:
+ boundary.side_effect = error
+ with patch(
+ "products.signals.backend.reviewer_pr_assignment.GitHubIntegration.first_for_team_repository",
+ return_value=github,
+ side_effect=error if step == "integration" else None,
+ ):
+ outcome = deliver_pull_request_assignment(team_id=team.id, report_id=str(report.id), pr_url=PR_URL)
+
+ assert outcome.status == "skipped"
+ assert not outcome.retryable
+ assert github.assignee_state == ["selected"]
+ assert report.artefacts.filter(type="pr_owner_assignment").count() == 1
+
+
+class TestFirstAssignmentRecovery:
+ def run_task(self, team_id: int, outcome: FirstAssignment, *, retries: int = 0) -> MagicMock:
+ task = assign_reviewers_on_implementation_pr
+ task.push_request(retries=retries)
+ try:
+ with (
+ patch("products.signals.backend.tasks.deliver_pull_request_assignment", return_value=outcome),
+ patch.object(task, "retry", side_effect=Retry()) as retry,
+ ):
+ if outcome.retryable and retries < task.max_retries:
+ with pytest.raises(Retry):
+ task.run(team_id=team_id, report_id="report", pr_url=PR_URL)
+ else:
+ task.run(team_id=team_id, report_id="report", pr_url=PR_URL)
+ finally:
+ task.pop_request()
+ return retry
+
+ @pytest.mark.parametrize(
+ "status,countdowns",
+ [
+ ("assigned", []),
+ ("skipped", []),
+ ("permanent_failure", []),
+ ("retryable_failure", [60, 120, 240, 480, 900]),
+ ("unknown_write", [60, 120, 240, 480, 900]),
+ ],
+ )
+ def test_recovery_is_bounded_and_only_for_retryable_outcomes(self, org_and_team, status, countdowns):
+ _, team = org_and_team
+ outcome = FirstAssignment(status=status, reason="test")
+
+ for attempt, countdown in enumerate(countdowns):
+ self.run_task(team.id, outcome, retries=attempt).assert_called_once_with(countdown=countdown)
+ self.run_task(team.id, outcome, retries=len(countdowns)).assert_not_called()
+
+ @pytest.mark.parametrize("retry_after,countdown", [(30, 900), (3600, 3600)])
+ def test_recovery_waits_for_the_rate_limit(
+ self, org_and_team: tuple[Organization, Team], retry_after: int, countdown: int
+ ) -> None:
+ _, team = org_and_team
+ outcome = FirstAssignment(status="retryable_failure", reason="rate_limited", retry_after=retry_after)
+
+ self.run_task(team.id, outcome, retries=4).assert_called_once_with(countdown=countdown)