:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
"""Shared Temporal trigger for on-demand scanner applications (observe and retry)."""
import enum
from uuid import UUID
from django.conf import settings
import structlog
from asgiref.sync import async_to_sync
from rest_framework.exceptions import Throttled
from temporalio.common import SearchAttributePair, TypedSearchAttributes
from temporalio.exceptions import WorkflowAlreadyStartedError
from posthog.exceptions import QuotaLimitExceeded
from posthog.temporal.common.client import sync_connect
from posthog.temporal.common.search_attributes import (
POSTHOG_SCANNER_ID_KEY,
POSTHOG_SESSION_RECORDING_ID_KEY,
POSTHOG_TEAM_ID_KEY,
)
from products.replay_vision.backend.enqueue_claims import (
pending_enqueue_claims_for_scanner,
pending_enqueue_claims_for_team,
release_enqueue_claim,
try_claim_enqueue_slot,
)
from products.replay_vision.backend.models.replay_observation import (
TERMINAL_STATUSES,
ObservationTrigger,
ReplayObservation,
)
from products.replay_vision.backend.models.replay_scanner import ReplayScanner
from products.replay_vision.backend.quota import QuotaState, compute_scanner_budget, quota_state
from products.replay_vision.backend.temporal.constants import (
APPLY_SCANNER_EXECUTION_TIMEOUT,
APPLY_SCANNER_WORKFLOW_NAME,
MAX_IN_FLIGHT_APPLIES_PER_SCANNER,
MAX_IN_FLIGHT_APPLIES_PER_TEAM,
build_apply_scanner_workflow_id,
on_demand_priority,
)
from products.replay_vision.backend.temporal.metrics import record_scanner_limit_reached
from products.replay_vision.backend.temporal.types import ApplyScannerInputs
logger = structlog.get_logger(__name__)
class WorkflowStartOutcome(enum.Enum):
STARTED = "started"
# A workflow with our deterministic id is already running — the scan is effectively in progress.
ALREADY_RUNNING = "already_running"
# A settled observation already holds this (scanner, session) slot, so nothing was started.
ALREADY_SCANNED = "already_scanned"
# The atomic enqueue-slot claim was refused: the in-flight caps have no headroom.
CAPPED = "capped"
FAILED = "failed"
def check_team_in_flight_capacity(team_id: int) -> None:
"""Raise 429 when in-flight rows plus enqueued-but-not-yet-persisted claims hit the team cap."""
count = ReplayObservation.in_flight_for_team(team_id).count() + pending_enqueue_claims_for_team(team_id)
if count >= MAX_IN_FLIGHT_APPLIES_PER_TEAM:
raise Throttled(detail=f"This team already has {count} observations running. Try again in a few minutes.")
def check_observation_quota(organization_id: UUID, observation_credits: int) -> QuotaState:
"""Raise 402 when starting an observation of this credit cost would exceed the org's monthly limit.
Returns the org's spend, so a free-only scanner's check can reuse it instead of reading the ledger again.
"""
snapshot = quota_state(organization_id=organization_id)
if snapshot.would_exceed(observation_credits):
# would_exceed is only ever true when a limit is set, so credit_limit is non-None here.
assert snapshot.credit_limit is not None
raise QuotaLimitExceeded(
detail=(
f"Starting this observation would exceed your monthly Replay vision limit of "
f"${snapshot.credit_limit / 100:,.2f}. Resets {snapshot.period_end.strftime('%b')} "
f"{snapshot.period_end.day}."
)
)
return snapshot
def check_scanner_quota(scanner: ReplayScanner, org_state: QuotaState | None = None) -> None:
"""Raise 402 when the scanner's own limit, or a free-only scanner's free credits, leave no room for another observation."""
if not scanner.has_spend_ceiling:
return
budget = compute_scanner_budget(scanner, org_state=org_state)
# blocked is only true when a limit is set; the direct check narrows without an assert.
if budget.credit_limit is not None and budget.blocked:
record_scanner_limit_reached("on_demand")
if budget.free_credits_bound:
raise QuotaLimitExceeded(
detail=(
f"This scanner only uses free credits, and your organization has used all {budget.credit_limit:,} "
f'for this billing period. To keep scanning now, turn off "Free credits only" in its settings.'
),
code="scanner_free_credits_used",
)
raise QuotaLimitExceeded(
detail=(
f"This scanner has {budget.remaining:,} of its {budget.credit_limit:,} credit limit left "
f"for this billing period, not enough for another observation. Raise the scanner's limit to keep "
f"scanning."
),
code="scanner_credit_limit_exceeded",
)
def _admission_still_within_caps(scanner: ReplayScanner) -> bool:
"""Validate a fresh claim against fresh counts. The claim is already registered, so this read
sees every competitor, and rows younger than the decay grace are still covered by their claims;
a pre-claim row snapshot staler than the grace self-corrects here instead of over-admitting."""
in_flight = ReplayObservation.in_flight_for_team(scanner.team_id)
if in_flight.count() + pending_enqueue_claims_for_team(scanner.team_id) > MAX_IN_FLIGHT_APPLIES_PER_TEAM:
return False
scanner_rows = in_flight.filter(scanner_id=scanner.id).count()
return scanner_rows + pending_enqueue_claims_for_scanner(scanner.id) <= MAX_IN_FLIGHT_APPLIES_PER_SCANNER
def claim_apply_scanner_slot(
scanner: ReplayScanner,
session_id: str,
*,
team_in_flight_rows: int | None = None,
scanner_in_flight_rows: int | None = None,
) -> tuple[str, bool]:
"""Claim the enqueue slot for one (scanner, session) ahead of the workflow start; on success the
caller owns the claim and must either pass `slot_already_claimed=True` to
`start_apply_scanner_workflow` or release it."""
workflow_id = build_apply_scanner_workflow_id(scanner.id, session_id)
if team_in_flight_rows is None:
team_in_flight_rows = ReplayObservation.in_flight_for_team(scanner.team_id).count()
if scanner_in_flight_rows is None:
scanner_in_flight_rows = (
ReplayObservation.in_flight_for_team(scanner.team_id).filter(scanner_id=scanner.id).count()
)
if not try_claim_enqueue_slot(
team_id=scanner.team_id,
scanner_id=scanner.id,
workflow_id=workflow_id,
team_in_flight_rows=team_in_flight_rows,
scanner_in_flight_rows=scanner_in_flight_rows,
):
return workflow_id, False
if not _admission_still_within_caps(scanner):
release_enqueue_claim(team_id=scanner.team_id, scanner_id=scanner.id, workflow_id=workflow_id)
return workflow_id, False
return workflow_id, True
def _is_already_scanned(scanner: ReplayScanner, session_id: str, finished_sessions: frozenset[str] | None) -> bool:
if finished_sessions is not None:
return session_id in finished_sessions
return ReplayObservation.objects.filter(
scanner_id=scanner.id, session_id=session_id, status__in=TERMINAL_STATUSES
).exists()
def start_apply_scanner_workflow(
scanner: ReplayScanner,
session_id: str,
*,
triggered_by_user_id: int | None,
trigger: ObservationTrigger,
team_in_flight_rows: int | None = None,
scanner_in_flight_rows: int | None = None,
finished_sessions: frozenset[str] | None = None,
slot_already_claimed: bool = False,
) -> tuple[str, WorkflowStartOutcome]:
"""Start the deterministic apply-scanner workflow for one (scanner, session); never raises.
An atomic enqueue-slot claim guards the in-flight caps; pass row counts to save two queries."""
# A settled observation holds the (scanner, session) slot for good, so starting a workflow would
# burn a run only to lose the INSERT in create_observation and hand back the row we can already
# see. Checked here rather than in each caller so a new one gets it by default; batch callers pass
# `finished_sessions` to answer it for the whole batch in one query, the same way they pass row
# counts. Skipped when the caller pre-claimed a slot: that is retry, which deleted the settled row
# to free this slot and means to scan again. Returning before the claim also keeps this exit from
# having a claim to leak.
if not slot_already_claimed and _is_already_scanned(scanner, session_id, finished_sessions):
return build_apply_scanner_workflow_id(scanner.id, session_id), WorkflowStartOutcome.ALREADY_SCANNED
if slot_already_claimed:
workflow_id = build_apply_scanner_workflow_id(scanner.id, session_id)
else:
workflow_id, claimed = claim_apply_scanner_slot(
scanner,
session_id,
team_in_flight_rows=team_in_flight_rows,
scanner_in_flight_rows=scanner_in_flight_rows,
)
if not claimed:
return workflow_id, WorkflowStartOutcome.CAPPED
try:
client = sync_connect()
async_to_sync(client.start_workflow)( # type: ignore[misc]
APPLY_SCANNER_WORKFLOW_NAME, # type: ignore[arg-type]
ApplyScannerInputs( # type: ignore[arg-type]
scanner_id=scanner.id,
session_id=session_id,
team_id=scanner.team_id,
triggered_by=trigger,
triggered_by_user_id=triggered_by_user_id,
),
id=workflow_id,
task_queue=settings.REPLAY_VISION_TASK_QUEUE,
execution_timeout=APPLY_SCANNER_EXECUTION_TIMEOUT,
# Every caller of this trigger is user-initiated (observe, bulk, inline, retry), so all runs qualify.
priority=on_demand_priority(scanner.team_id),
# Stamp the scanner id so on-demand applies count toward the sweep's in-flight cap.
search_attributes=TypedSearchAttributes(
search_attributes=[
SearchAttributePair(key=POSTHOG_TEAM_ID_KEY, value=scanner.team_id),
SearchAttributePair(key=POSTHOG_SESSION_RECORDING_ID_KEY, value=session_id),
SearchAttributePair(key=POSTHOG_SCANNER_ID_KEY, value=str(scanner.id)),
]
),
)
except WorkflowAlreadyStartedError as exc:
# Pin to our own workflow_id so a future id_reuse_policy change can't silently accept an unrelated run.
if exc.workflow_id != workflow_id:
logger.exception("replay_vision.observe.workflow_id_mismatch", workflow_id=workflow_id)
release_enqueue_claim(team_id=scanner.team_id, scanner_id=scanner.id, workflow_id=workflow_id)
return workflow_id, WorkflowStartOutcome.FAILED
logger.info("replay_vision.observe.workflow_already_started", workflow_id=workflow_id)
# A persisted row already counts this run, so our claim is a resurrected duplicate — drop
# it. Without a row the running workflow is still in the enqueue gap: keep the claim.
if ReplayObservation.objects.filter(scanner_id=scanner.id, session_id=session_id).exists():
release_enqueue_claim(
team_id=scanner.team_id, scanner_id=scanner.id, workflow_id=workflow_id, immediately=True
)
return workflow_id, WorkflowStartOutcome.ALREADY_RUNNING
except Exception:
logger.exception("replay_vision.observe.workflow_start_failed", workflow_id=workflow_id)
release_enqueue_claim(team_id=scanner.team_id, scanner_id=scanner.id, workflow_id=workflow_id)
return workflow_id, WorkflowStartOutcome.FAILED
return workflow_id, WorkflowStartOutcome.STARTED