:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import random
from datetime import timedelta
from typing import Any, Optional
from temporalio import activity
from temporalio.exceptions import ApplicationError
from posthog.exceptions_capture import capture_exception
from posthog.temporal.common.errors import NonReportableError
from products.tasks.backend.facade.compute_quota import ComputeBillingLimitExceeded
SANDBOX_RATE_LIMIT_BASE_DELAY_SECONDS = 45
SANDBOX_RATE_LIMIT_MAX_DELAY_SECONDS = 300
TEMPORAL_TIMEOUT_ERROR_TYPES = frozenset({"start_to_close", "schedule_to_start", "schedule_to_close", "heartbeat"})
def sandbox_rate_limit_retry_delay(attempt: int) -> float:
ceiling = min(
SANDBOX_RATE_LIMIT_BASE_DELAY_SECONDS * 2 ** max(attempt - 1, 0),
SANDBOX_RATE_LIMIT_MAX_DELAY_SECONDS,
)
return random.uniform(ceiling / 2, ceiling)
class ProcessTaskError(ApplicationError):
def __init__(
self, message: str, context: dict[str, Any], cause: Optional[BaseException], capture: bool = True, **kwargs
):
self.context = context or {}
if "team" not in self.context:
self.context["team"] = "array"
# `capture=False` skips error-tracking capture for expected, recoverable failures
# (e.g. transient infra timeouts that Temporal retries) so they don't create noisy issues.
if cause is not None and capture:
capture_exception(cause, self.context)
# Retry policies match non_retryable_error_types against this type; the SDK omits it unless set.
kwargs.setdefault("type", type(self).__name__)
super().__init__(message, self.context, **kwargs)
class ProcessTaskFatalError(ProcessTaskError):
"""Fatal errors that should not be retried."""
def __init__(self, message: str, context: dict[str, Any], cause: Exception, **kwargs):
super().__init__(message, context, cause, non_retryable=True, **kwargs)
class ProcessTaskTransientError(ProcessTaskError):
"""Transient errors that may succeed on retry."""
def __init__(self, message: str, context: dict[str, Any], cause: BaseException, **kwargs):
super().__init__(message, context, cause, non_retryable=False, **kwargs)
def is_transient_failure_error_type(error_type: str | None) -> bool:
if error_type in TEMPORAL_TIMEOUT_ERROR_TYPES:
return True
pending = [ProcessTaskTransientError]
while pending:
error_class = pending.pop()
if error_class.__name__ == error_type:
return True
pending.extend(error_class.__subclasses__())
return False
class TaskNotFoundError(ProcessTaskFatalError):
pass
class TaskRunNotReadyError(ProcessTaskTransientError):
"""The TaskRun row is not yet visible to this activity.
Typically the creating transaction has not committed by the time the activity's
first read runs (the run was created inside an enclosing transaction.atomic block).
Retryable so the activity's existing retry policy recovers once the row is visible.
Intentionally does not capture to error tracking — this is an expected transient
window the retry absorbs, not a fault worth an issue.
"""
def __init__(self, message: str, context: dict[str, Any]):
# Bypass ProcessTaskTransientError.__init__ to pass cause=None, which skips the
# capture_exception() call in ProcessTaskError — avoiding error-tracking noise.
ProcessTaskError.__init__(self, message, context, None, non_retryable=False)
class GitHubRateLimitedError(ProcessTaskTransientError):
"""GitHub rate-limited a call (or our egress budget shed it before it was sent).
An expected, recoverable condition rather than a fault. Kept retryable so the
activity's retry policy recovers once the limit window passes, but intentionally
not captured to error tracking — there is nothing to investigate, and capturing
it mints a noisy issue for something we expect to happen. ``retry_after`` (seconds)
drives ``next_retry_delay`` so the retry lands after the window instead of burning
every attempt inside it, and is folded into the message so the surfaced error names
a real wait instead of an empty reset time.
"""
def __init__(self, message: str, context: dict[str, Any], retry_after: int):
# Bypass ProcessTaskTransientError.__init__ to pass cause=None with capture=False,
# skipping the capture_exception() call in ProcessTaskError (mirrors TaskRunNotReadyError).
ProcessTaskError.__init__(
self,
message,
context,
None,
capture=False,
non_retryable=False,
next_retry_delay=timedelta(seconds=retry_after),
)
class TaskInvalidStateError(ProcessTaskFatalError):
pass
class OrganizationExecutionError(ProcessTaskError):
def __init__(self, message: str, team_id: int, reason: str) -> None:
super().__init__(message, {"team_id": team_id, "reason": reason}, None, capture=False, non_retryable=True)
class SandboxProvisionError(ProcessTaskTransientError):
"""Failed to provision sandbox environment."""
pass
class ComputeBillingLimitError(ProcessTaskError, ComputeBillingLimitExceeded):
def __init__(self, context: dict[str, Any], reason: str = "posthog_code_billing_limit_exceeded"):
from products.tasks.backend.logic.services.compute_quota import ORGANIZATION_DEACTIVATED_DENIAL_CODE
self.reason = reason
message = (
"Your organization has been deactivated."
if reason == ORGANIZATION_DEACTIVATED_DENIAL_CODE
else "Your organization reached its PostHog Desktop usage limit."
)
super().__init__(
message,
{**context, "reason": reason},
None,
capture=False,
non_retryable=True,
)
class ActiveSessionCapError(ProcessTaskError):
def __init__(self, context: dict[str, Any], scope: str, message: str):
self.scope = scope
super().__init__(
message,
{**context, "scope": scope},
None,
capture=False,
non_retryable=True,
)
class SandboxNetworkPolicyError(ProcessTaskFatalError):
pass
class SandboxNotFoundError(ProcessTaskFatalError):
"""Sandbox does not exist."""
pass
class SandboxExecutionError(ProcessTaskTransientError):
"""Error during sandbox command execution."""
pass
class SandboxControlPlaneError(SandboxExecutionError):
"""The sandbox control plane refused or failed a call, before the sandbox itself saw it.
The fault belongs to the control plane, not to the run, so these stay retryable and are
never captured to error tracking. Capturing one mints an issue nobody can act on, and a
fresh one per provider message: the cause is constructed rather than raised, so every
occurrence carries its own synthesized traceback and none of them group.
"""
def __init__(self, message: str, context: dict[str, Any], retry_delay: Optional[timedelta] = None):
ProcessTaskError.__init__(
self,
message,
context,
None,
capture=False,
non_retryable=False,
next_retry_delay=retry_delay,
)
class SandboxRateLimitedError(SandboxControlPlaneError):
"""The egress proxy in front of the sandbox control plane shed the call.
Backs off past the limit window so the retry lands after it rather than inside it.
"""
def __init__(self, message: str, context: dict[str, Any]):
attempt = activity.info().attempt if activity.in_activity() else 1
super().__init__(message, context, retry_delay=timedelta(seconds=sandbox_rate_limit_retry_delay(attempt)))
class SandboxControlPlaneUnavailableError(SandboxControlPlaneError):
"""The sandbox control plane answered a call with a gateway status (5xx).
The activity's own retry policy is the right backoff here. Unlike a rate limit there is
no window to wait out, so this does not set a delay of its own.
"""
class SandboxMissingRepositoryError(ProcessTaskFatalError):
"""The repository directory the agent-server needs as its cwd is absent from the sandbox.
Happens when a run reaches agent-server start without a clone — no snapshot restored and no
usable GitHub credentials. Retrying cannot make the directory appear, so fail immediately
with the real reason instead of burning health-check timeouts on a server that can never
open a session.
"""
pass
class SandboxNotRunningError(SandboxExecutionError):
"""Sandbox is not in a running state."""
pass
class SandboxTimeoutError(ProcessTaskTransientError):
"""Sandbox operation timed out."""
pass
class SandboxCleanupError(ProcessTaskTransientError):
"""Error during sandbox cleanup/destruction."""
pass
class SnapshotNotFoundError(ProcessTaskTransientError):
"""Snapshot does not exist."""
pass
class SnapshotNotReadyError(ProcessTaskTransientError):
"""Snapshot exists but is not ready for use."""
pass
class SnapshotCreationError(ProcessTaskTransientError):
"""Failed to create snapshot."""
pass
class SnapshotTimeoutError(ProcessTaskTransientError):
"""Transient timeout/connection error while creating a snapshot; safe to retry."""
pass
class SnapshotFileLimitExceededError(ProcessTaskFatalError):
"""Modal refuses to snapshot a directory/filesystem holding more than its hard file-count
cap (1,000,000 files). This is a permanent limit, not a transient blip, so retrying the same
snapshot cannot succeed — the caller must shrink the tree (prune node_modules, virtualenvs,
and package caches) before it can snapshot.
Non-retryable and captured to error tracking: it is a named, classified condition (not the
generic mystery issue this replaced), and callers with no fallback — the dev-stack image bake,
the standalone snapshot activity — need it visible. The resume path recovers by pruning and
letting Temporal retry, so it converts this into a transient error itself rather than swallowing
the signal.
"""
pass
class RepositoryCloneError(ProcessTaskTransientError):
"""Failed to clone repository."""
pass
class RepositoryAccessError(ProcessTaskFatalError, NonReportableError):
"""GitHub refused to serve the repository with the credential the sandbox has.
The run has no token, or its token does not cover the repository. A retry reuses the same
credential, so it cannot succeed. This is an expected customer setup state, so it is not
captured to error tracking.
"""
def __init__(self, message: str, context: dict[str, Any]):
ProcessTaskError.__init__(self, message, context, None, capture=False, non_retryable=True)
class RetryableRepositorySetupError(ProcessTaskTransientError):
"""Failed to setup repository (install dependencies, etc)."""
pass
class GitHubIntegrationError(ProcessTaskFatalError):
"""GitHub integration not found or invalid."""
pass
class GitHubAuthenticationError(ProcessTaskFatalError):
"""Failed to authenticate with GitHub."""
pass
class CredentialUnavailableError(ProcessTaskFatalError, NonReportableError):
"""A sandbox credential can never be resolved again for this run — the backing
integration row was deleted mid-run or the user must re-authorize.
Not retriable, and an expected customer-initiated state rather than a systemic
failure, so it is not captured to error tracking.
"""
def __init__(self, message: str, context: dict[str, Any], cause: Exception | None = None):
ProcessTaskError.__init__(self, message, context, cause, capture=False, non_retryable=True)
class GatewayTokenUnavailableError(ProcessTaskFatalError, NonReportableError):
"""No Go token for a posthog_code run, for a reason a retry won't change. Expected customer
state, so not captured."""
def __init__(self, message: str, context: dict[str, Any]):
ProcessTaskError.__init__(self, message, context, None, capture=False, non_retryable=True)
class PersonalAPIKeyError(ProcessTaskTransientError):
"""Failed to create or inject personal API key."""
pass
class OAuthTokenError(ProcessTaskTransientError):
"""Failed to create OAuth access token."""
pass
class TaskExecutionFailedError(ProcessTaskError):
"""Task execution completed but with non-zero exit code."""
def __init__(
self,
message: str,
exit_code: int,
stdout: str = "",
stderr: str = "",
context: Optional[dict[str, Any]] = None,
cause: Optional[Exception] = None,
non_retryable: bool = False,
):
self.exit_code = exit_code
self.stdout = stdout
self.stderr = stderr
if cause is None:
cause = RuntimeError(f"Task failed with exit code {exit_code}")
super().__init__(message, context or {}, cause, non_retryable=non_retryable)