Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

11.2 KB
Raw Blame
"""Quarantined identifiers: list, add, lift, expire."""

from __future__ import annotations

from collections import defaultdict
from datetime import datetime, timedelta
from uuid import UUID

from django.db import transaction
from django.db.models import Q, QuerySet
from django.utils import timezone

from ..db import READER_DB, WRITER_DB
from ..facade.contracts import AGENT_QUARANTINE_MAX_DAYS, FLAKINESS_EXPIRY_SOON_DAYS
from ..facade.enums import ActorType
from ..models import QuarantinedIdentifier, Repo, Run
from . import errors, github_api, repos
from .run_queries import SnapshotKey

# How long a lift stays scoped to its commit. A branch that forked before the lift and is still
# open after this has usually merged the default branch since; past it, the lift applies everywhere.
LIFT_SCOPE_DAYS = 30


def list_quarantined_identifiers(
    repo_id: UUID, team_id: int, identifier: str | None = None, run_type: str | None = None
) -> QuerySet[QuarantinedIdentifier]:
    qs = (
        QuarantinedIdentifier.objects.using(WRITER_DB)
        .filter(repo_id=repo_id, team_id=team_id)
        # Preload `source_run` so the facade can render the "what was wrong"
        # link without an extra fetch per row. `Run.metadata` (JSONField) and
        # `Run.error_message` (TextField) can be large and aren't needed for
        # the summary — defer to keep response payloads tight.
        .select_related("source_run")
        .defer("source_run__metadata", "source_run__error_message")
    )
    if run_type:
        qs = qs.filter(run_type=run_type)
    if identifier:
        qs = qs.filter(identifier=identifier)
    else:
        now = timezone.now()
        qs = qs.filter(Q(expires_at__isnull=True) | Q(expires_at__gt=now))
    return qs.order_by("-created_at")


def active_quarantined_identifiers(
    repo_id: UUID, team_id: int, run_type: str, using: str
) -> QuerySet[QuarantinedIdentifier, str]:
    """Identifiers under an active quarantine, as a subquery that filters snapshot rows in SQL.

    `using` must name the database of the outer query, because Django refuses a subquery
    across database aliases.
    """
    return (
        QuarantinedIdentifier.objects.using(using)
        .filter(repo_id=repo_id, team_id=team_id, run_type=run_type)
        .filter(Q(expires_at__isnull=True) | Q(expires_at__gt=timezone.now()))
        .values_list("identifier", flat=True)
    )


def expiry_soon_cutoff(now: datetime, within_days: int = FLAKINESS_EXPIRY_SOON_DAYS) -> datetime:
    """The moment past which an expiry is close enough that somebody has to decide about it."""
    return now + timedelta(days=within_days)


def is_expiring_soon(entry: QuarantinedIdentifier, cutoff: datetime) -> bool:
    """Whether an active quarantine runs out inside the window.

    An entry with no expiry never does. It is a standing exception, and nothing about it changes
    on its own, so there is no date for a reminder to hang off.
    """
    return entry.expires_at is not None and entry.expires_at <= cutoff


def list_expiring_quarantines(
    repo_id: UUID, *, now: datetime, within_days: int = FLAKINESS_EXPIRY_SOON_DAYS
) -> list[QuarantinedIdentifier]:
    """Active quarantines that run out inside the window, soonest first.

    An entry leaves this list when somebody extends it past the window, lifts it, or lets it lapse.
    A caller that reads this on a schedule widens `within_days`, so an entry cannot expire in the
    gap between two of its runs.
    """
    return list(
        QuarantinedIdentifier.objects.using(READER_DB)
        .filter(repo_id=repo_id, expires_at__gt=now, expires_at__lte=expiry_soon_cutoff(now, within_days))
        # Preload `source_run` so the caller can render the "what was wrong" link without an extra
        # fetch per row. `Run.metadata` (JSONField) and `Run.error_message` (TextField) can be
        # large and aren't needed for the summary.
        .select_related("source_run")
        .defer("source_run__metadata", "source_run__error_message")
        .order_by("expires_at")
    )


def active_quarantine_keys(repo_id: UUID, *, now: datetime) -> set[SnapshotKey]:
    """Every identity under a quarantine that has not run out, whatever its expiry."""
    return {
        SnapshotKey(run_type=run_type, identifier=identifier)
        for run_type, identifier in QuarantinedIdentifier.objects.using(READER_DB)
        .filter(repo_id=repo_id)
        .filter(Q(expires_at__isnull=True) | Q(expires_at__gt=now))
        .values_list("run_type", "identifier")
    }


@transaction.atomic(using=WRITER_DB)
def quarantine_identifier(
    repo_id: UUID,
    identifier: str,
    run_type: str,
    reason: str,
    user_id: int,
    team_id: int,
    expires_at: datetime | None = None,
    source_run_id: UUID | None = None,
    source: ActorType = ActorType.HUMAN,
    notify_owners: bool = False,
) -> QuarantinedIdentifier:
    repos.get_repo(repo_id, team_id)  # raises RepoNotFoundError if repo not owned by team
    now = timezone.now()
    if source == ActorType.AGENT:
        latest = now + timedelta(days=AGENT_QUARANTINE_MAX_DAYS)
        if expires_at is None or expires_at > latest:
            expires_at = latest
    # Resolve the source run inside the team scope so a malicious caller can't
    # attach a quarantine to an unrelated run. Silently drop on mismatch — the
    # quarantine itself still wins; we just lose the "what was wrong" pointer.
    # We fetch (not just .exists()) so the facade can serialize source_run
    # without a lazy-load on the freshly-created row.
    # The source run is locked first, in the same order as the retention sweep,
    # so a run the sweep deletes meanwhile resolves to None instead of failing
    # the insert.
    with transaction.atomic(using=WRITER_DB):
        source_run: Run | None = None
        if source_run_id is not None:
            source_run = (
                Run.objects.using(WRITER_DB)
                .select_for_update()
                .filter(id=source_run_id, repo_id=repo_id, team_id=team_id)
                .first()
            )
        QuarantinedIdentifier.objects.using(WRITER_DB).select_for_update().filter(
            repo_id=repo_id,
            identifier=identifier,
            run_type=run_type,
            team_id=team_id,
        ).filter(Q(expires_at__isnull=True) | Q(expires_at__gt=now)).update(expires_at=now)
        entry = QuarantinedIdentifier.objects.using(WRITER_DB).create(
            repo_id=repo_id,
            identifier=identifier,
            run_type=run_type,
            team_id=team_id,
            reason=reason,
            expires_at=expires_at,
            created_by_id=user_id,
            source_run=source_run,
            source=source,
        )
    if notify_owners:
        from ..tasks.tasks import (  # noqa: PLC0415 — avoids the logic/tasks circular import
            QUARANTINE_NOTICE_EXPIRY_SECONDS,
            notify_quarantine_owners,
        )

        entry_id = str(entry.id)
        transaction.on_commit(
            lambda: notify_quarantine_owners.apply_async(
                args=(team_id, entry_id), expires=QUARANTINE_NOTICE_EXPIRY_SECONDS
            ),
            using=WRITER_DB,
            robust=True,
        )
    return entry


def _lift_commit(repo: Repo) -> str:
    # A lift without a commit would apply to every branch, including branches that forked before
    # the fix and still render the old picture. Refuse it, so the quarantine stays.
    lifted_at_sha = github_api.default_branch_head_sha(repo)
    if lifted_at_sha is None:
        raise errors.LiftCommitUnknownError(
            "GitHub cannot name the default branch head, so the lift cannot be scoped to a commit. "
            "The quarantine stays. Try again in a minute."
        )
    return lifted_at_sha


def unquarantine_identifier(repo_id: UUID, identifier: str, run_type: str, team_id: int) -> None:
    repo = repos.get_repo(repo_id, team_id)  # raises RepoNotFoundError if repo not owned by team
    # Take the cutoff before the GitHub request, and lift only rows that existed at the cutoff. A
    # quarantine that somebody creates while the request is in flight must survive this lift.
    now = timezone.now()
    active = (
        QuarantinedIdentifier.objects.using(WRITER_DB)
        .filter(
            repo_id=repo_id,
            identifier=identifier,
            run_type=run_type,
            team_id=team_id,
            created_at__lte=now,
        )
        .filter(Q(expires_at__isnull=True) | Q(expires_at__gt=now))
    )
    # A repeated lift stays a no-op, even while GitHub cannot name the head.
    if not active.exists():
        return
    active.update(expires_at=now, lifted_at_sha=_lift_commit(repo))


def expire_quarantine_entry(entry_id: UUID, team_id: int) -> None:
    now = timezone.now()
    active = Q(expires_at__isnull=True) | Q(expires_at__gt=now)
    try:
        entry = QuarantinedIdentifier.objects.using(WRITER_DB).filter(active).get(id=entry_id, team_id=team_id)
    except QuarantinedIdentifier.DoesNotExist as e:
        raise errors.RunNotFoundError(f"Quarantine entry {entry_id} not found or already expired") from e

    # The cutoff is `now`, taken before the GitHub request, for the same reason as in
    # `unquarantine_identifier`.
    lifted_at_sha = _lift_commit(entry.repo)
    # Expire all active entries for the same identifier/run_type, not just this one
    QuarantinedIdentifier.objects.using(WRITER_DB).filter(
        repo_id=entry.repo_id,
        identifier=entry.identifier,
        run_type=entry.run_type,
        team_id=team_id,
        created_at__lte=now,
    ).filter(active).update(expires_at=now, lifted_at_sha=lifted_at_sha)


def identifiers_lifted_after_commit(run: Run, *, now: datetime) -> set[str]:
    """Identifiers in `run` whose quarantine was lifted at a commit that `run`'s commit does not contain.

    Such a run is on a branch that forked before the lift. The quarantine still applies there,
    because the branch lacks what the lift relied on. It stops applying once the branch merges the
    default branch. Only the latest quarantine event of an identifier counts, so a lift that a
    newer quarantine replaced no longer applies. A lift whose ancestry GitHub cannot confirm keeps
    applying the quarantine, because a missed gate costs less than a red run that nobody on the
    branch can fix.
    """
    latest_events = (
        QuarantinedIdentifier.objects.using(WRITER_DB)
        .filter(
            repo_id=run.repo_id,
            run_type=run.run_type,
            team_id=run.team_id,
            identifier__in=run.snapshots.using(WRITER_DB).values("identifier"),
        )
        .order_by("identifier", "-created_at")
        .distinct("identifier")
        .values_list("identifier", "lifted_at_sha", "expires_at")
    )
    scope_start = now - timedelta(days=LIFT_SCOPE_DAYS)
    identifiers_by_sha: dict[str, set[str]] = defaultdict(set)
    for identifier, lifted_at_sha, expires_at in latest_events:
        if lifted_at_sha is not None and expires_at is not None and scope_start < expires_at <= now:
            identifiers_by_sha[lifted_at_sha].add(identifier)

    return {
        identifier
        for lifted_at_sha, identifiers in identifiers_by_sha.items()
        if not github_api.commit_contains(run.repo, lifted_at_sha, run.commit_sha)
        for identifier in identifiers
    }