Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

11.5 KB
Raw Blame
"""Shared Temporal trigger for on-demand scanner applications (observe and retry)."""

import enum
from uuid import UUID

from django.conf import settings

import structlog
from asgiref.sync import async_to_sync
from rest_framework.exceptions import Throttled
from temporalio.common import SearchAttributePair, TypedSearchAttributes
from temporalio.exceptions import WorkflowAlreadyStartedError

from posthog.exceptions import QuotaLimitExceeded
from posthog.temporal.common.client import sync_connect
from posthog.temporal.common.search_attributes import (
    POSTHOG_SCANNER_ID_KEY,
    POSTHOG_SESSION_RECORDING_ID_KEY,
    POSTHOG_TEAM_ID_KEY,
)

from products.replay_vision.backend.enqueue_claims import (
    pending_enqueue_claims_for_scanner,
    pending_enqueue_claims_for_team,
    release_enqueue_claim,
    try_claim_enqueue_slot,
)
from products.replay_vision.backend.models.replay_observation import (
    TERMINAL_STATUSES,
    ObservationTrigger,
    ReplayObservation,
)
from products.replay_vision.backend.models.replay_scanner import ReplayScanner
from products.replay_vision.backend.quota import QuotaState, compute_scanner_budget, quota_state
from products.replay_vision.backend.temporal.constants import (
    APPLY_SCANNER_EXECUTION_TIMEOUT,
    APPLY_SCANNER_WORKFLOW_NAME,
    MAX_IN_FLIGHT_APPLIES_PER_SCANNER,
    MAX_IN_FLIGHT_APPLIES_PER_TEAM,
    build_apply_scanner_workflow_id,
    on_demand_priority,
)
from products.replay_vision.backend.temporal.metrics import record_scanner_limit_reached
from products.replay_vision.backend.temporal.types import ApplyScannerInputs

logger = structlog.get_logger(__name__)


class WorkflowStartOutcome(enum.Enum):
    STARTED = "started"
    # A workflow with our deterministic id is already running — the scan is effectively in progress.
    ALREADY_RUNNING = "already_running"
    # A settled observation already holds this (scanner, session) slot, so nothing was started.
    ALREADY_SCANNED = "already_scanned"
    # The atomic enqueue-slot claim was refused: the in-flight caps have no headroom.
    CAPPED = "capped"
    FAILED = "failed"


def check_team_in_flight_capacity(team_id: int) -> None:
    """Raise 429 when in-flight rows plus enqueued-but-not-yet-persisted claims hit the team cap."""
    count = ReplayObservation.in_flight_for_team(team_id).count() + pending_enqueue_claims_for_team(team_id)
    if count >= MAX_IN_FLIGHT_APPLIES_PER_TEAM:
        raise Throttled(detail=f"This team already has {count} observations running. Try again in a few minutes.")


def check_observation_quota(organization_id: UUID, observation_credits: int) -> QuotaState:
    """Raise 402 when starting an observation of this credit cost would exceed the org's monthly limit.

    Returns the org's spend, so a free-only scanner's check can reuse it instead of reading the ledger again.
    """
    snapshot = quota_state(organization_id=organization_id)
    if snapshot.would_exceed(observation_credits):
        # would_exceed is only ever true when a limit is set, so credit_limit is non-None here.
        assert snapshot.credit_limit is not None
        raise QuotaLimitExceeded(
            detail=(
                f"Starting this observation would exceed your monthly Replay vision limit of "
                f"${snapshot.credit_limit / 100:,.2f}. Resets {snapshot.period_end.strftime('%b')} "
                f"{snapshot.period_end.day}."
            )
        )
    return snapshot


def check_scanner_quota(scanner: ReplayScanner, org_state: QuotaState | None = None) -> None:
    """Raise 402 when the scanner's own limit, or a free-only scanner's free credits, leave no room for another observation."""
    if not scanner.has_spend_ceiling:
        return
    budget = compute_scanner_budget(scanner, org_state=org_state)
    # blocked is only true when a limit is set; the direct check narrows without an assert.
    if budget.credit_limit is not None and budget.blocked:
        record_scanner_limit_reached("on_demand")
        if budget.free_credits_bound:
            raise QuotaLimitExceeded(
                detail=(
                    f"This scanner only uses free credits, and your organization has used all {budget.credit_limit:,} "
                    f'for this billing period. To keep scanning now, turn off "Free credits only" in its settings.'
                ),
                code="scanner_free_credits_used",
            )
        raise QuotaLimitExceeded(
            detail=(
                f"This scanner has {budget.remaining:,} of its {budget.credit_limit:,} credit limit left "
                f"for this billing period, not enough for another observation. Raise the scanner's limit to keep "
                f"scanning."
            ),
            code="scanner_credit_limit_exceeded",
        )


def _admission_still_within_caps(scanner: ReplayScanner) -> bool:
    """Validate a fresh claim against fresh counts. The claim is already registered, so this read
    sees every competitor, and rows younger than the decay grace are still covered by their claims;
    a pre-claim row snapshot staler than the grace self-corrects here instead of over-admitting."""
    in_flight = ReplayObservation.in_flight_for_team(scanner.team_id)
    if in_flight.count() + pending_enqueue_claims_for_team(scanner.team_id) > MAX_IN_FLIGHT_APPLIES_PER_TEAM:
        return False
    scanner_rows = in_flight.filter(scanner_id=scanner.id).count()
    return scanner_rows + pending_enqueue_claims_for_scanner(scanner.id) <= MAX_IN_FLIGHT_APPLIES_PER_SCANNER


def claim_apply_scanner_slot(
    scanner: ReplayScanner,
    session_id: str,
    *,
    team_in_flight_rows: int | None = None,
    scanner_in_flight_rows: int | None = None,
) -> tuple[str, bool]:
    """Claim the enqueue slot for one (scanner, session) ahead of the workflow start; on success the
    caller owns the claim and must either pass `slot_already_claimed=True` to
    `start_apply_scanner_workflow` or release it."""
    workflow_id = build_apply_scanner_workflow_id(scanner.id, session_id)
    if team_in_flight_rows is None:
        team_in_flight_rows = ReplayObservation.in_flight_for_team(scanner.team_id).count()
    if scanner_in_flight_rows is None:
        scanner_in_flight_rows = (
            ReplayObservation.in_flight_for_team(scanner.team_id).filter(scanner_id=scanner.id).count()
        )
    if not try_claim_enqueue_slot(
        team_id=scanner.team_id,
        scanner_id=scanner.id,
        workflow_id=workflow_id,
        team_in_flight_rows=team_in_flight_rows,
        scanner_in_flight_rows=scanner_in_flight_rows,
    ):
        return workflow_id, False
    if not _admission_still_within_caps(scanner):
        release_enqueue_claim(team_id=scanner.team_id, scanner_id=scanner.id, workflow_id=workflow_id)
        return workflow_id, False
    return workflow_id, True


def _is_already_scanned(scanner: ReplayScanner, session_id: str, finished_sessions: frozenset[str] | None) -> bool:
    if finished_sessions is not None:
        return session_id in finished_sessions
    return ReplayObservation.objects.filter(
        scanner_id=scanner.id, session_id=session_id, status__in=TERMINAL_STATUSES
    ).exists()


def start_apply_scanner_workflow(
    scanner: ReplayScanner,
    session_id: str,
    *,
    triggered_by_user_id: int | None,
    trigger: ObservationTrigger,
    team_in_flight_rows: int | None = None,
    scanner_in_flight_rows: int | None = None,
    finished_sessions: frozenset[str] | None = None,
    slot_already_claimed: bool = False,
) -> tuple[str, WorkflowStartOutcome]:
    """Start the deterministic apply-scanner workflow for one (scanner, session); never raises.
    An atomic enqueue-slot claim guards the in-flight caps; pass row counts to save two queries."""
    # A settled observation holds the (scanner, session) slot for good, so starting a workflow would
    # burn a run only to lose the INSERT in create_observation and hand back the row we can already
    # see. Checked here rather than in each caller so a new one gets it by default; batch callers pass
    # `finished_sessions` to answer it for the whole batch in one query, the same way they pass row
    # counts. Skipped when the caller pre-claimed a slot: that is retry, which deleted the settled row
    # to free this slot and means to scan again. Returning before the claim also keeps this exit from
    # having a claim to leak.
    if not slot_already_claimed and _is_already_scanned(scanner, session_id, finished_sessions):
        return build_apply_scanner_workflow_id(scanner.id, session_id), WorkflowStartOutcome.ALREADY_SCANNED
    if slot_already_claimed:
        workflow_id = build_apply_scanner_workflow_id(scanner.id, session_id)
    else:
        workflow_id, claimed = claim_apply_scanner_slot(
            scanner,
            session_id,
            team_in_flight_rows=team_in_flight_rows,
            scanner_in_flight_rows=scanner_in_flight_rows,
        )
        if not claimed:
            return workflow_id, WorkflowStartOutcome.CAPPED
    try:
        client = sync_connect()
        async_to_sync(client.start_workflow)(  # type: ignore[misc]
            APPLY_SCANNER_WORKFLOW_NAME,  # type: ignore[arg-type]
            ApplyScannerInputs(  # type: ignore[arg-type]
                scanner_id=scanner.id,
                session_id=session_id,
                team_id=scanner.team_id,
                triggered_by=trigger,
                triggered_by_user_id=triggered_by_user_id,
            ),
            id=workflow_id,
            task_queue=settings.REPLAY_VISION_TASK_QUEUE,
            execution_timeout=APPLY_SCANNER_EXECUTION_TIMEOUT,
            # Every caller of this trigger is user-initiated (observe, bulk, inline, retry), so all runs qualify.
            priority=on_demand_priority(scanner.team_id),
            # Stamp the scanner id so on-demand applies count toward the sweep's in-flight cap.
            search_attributes=TypedSearchAttributes(
                search_attributes=[
                    SearchAttributePair(key=POSTHOG_TEAM_ID_KEY, value=scanner.team_id),
                    SearchAttributePair(key=POSTHOG_SESSION_RECORDING_ID_KEY, value=session_id),
                    SearchAttributePair(key=POSTHOG_SCANNER_ID_KEY, value=str(scanner.id)),
                ]
            ),
        )
    except WorkflowAlreadyStartedError as exc:
        # Pin to our own workflow_id so a future id_reuse_policy change can't silently accept an unrelated run.
        if exc.workflow_id != workflow_id:
            logger.exception("replay_vision.observe.workflow_id_mismatch", workflow_id=workflow_id)
            release_enqueue_claim(team_id=scanner.team_id, scanner_id=scanner.id, workflow_id=workflow_id)
            return workflow_id, WorkflowStartOutcome.FAILED
        logger.info("replay_vision.observe.workflow_already_started", workflow_id=workflow_id)
        # A persisted row already counts this run, so our claim is a resurrected duplicate — drop
        # it. Without a row the running workflow is still in the enqueue gap: keep the claim.
        if ReplayObservation.objects.filter(scanner_id=scanner.id, session_id=session_id).exists():
            release_enqueue_claim(
                team_id=scanner.team_id, scanner_id=scanner.id, workflow_id=workflow_id, immediately=True
            )
        return workflow_id, WorkflowStartOutcome.ALREADY_RUNNING
    except Exception:
        logger.exception("replay_vision.observe.workflow_start_failed", workflow_id=workflow_id)
        release_enqueue_claim(team_id=scanner.team_id, scanner_id=scanner.id, workflow_id=workflow_id)
        return workflow_id, WorkflowStartOutcome.FAILED
    return workflow_id, WorkflowStartOutcome.STARTED