Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

34.6 KB
Raw Blame
from collections.abc import Mapping
from typing import TYPE_CHECKING, Literal

import structlog
from prometheus_client import Counter, Gauge, Histogram

from posthog.metrics import pushed_metrics_registry

logger = structlog.get_logger(__name__)

if TYPE_CHECKING:
    from products.tasks.backend.models import TaskRun


TaskWorkflowStartOutcome = Literal["attempted", "blocked", "failed", "started"]
# Why a warm Run died unused. `released` is a deliberate hand-back (the composer cancels the run when
# the draft is abandoned) and is the cheap, expected miss; `idle_timeout` means nobody released it and
# it sat until the workflow reclaimed it, which is the wasteful one to drive down.
PrewarmedUnusedReason = Literal["released", "idle_timeout", "other"]
CustomImageBuildOutcome = Literal["started", "succeeded", "failed", "scan_rejected"]
DevStackImageBakeOutcome = Literal["succeeded", "bake_failed", "failed", "dispatch_failed"]
# Outcome of an SSE task-run stream connection when it closes.
#   completed         — stream reached its completion sentinel
#   stream_error      — Redis/stream error sentinel ended the connection
#   unavailable       — stream key never appeared within the wait timeout
#   drained           — terminal run whose stream key already expired; ended immediately
#   client_disconnect — client went away (GeneratorExit) before completion
#   rotated           — per-connection cap reached; clean EOF, client resumes
#   backlog_error     — run-log read for connect-time backlog failed; client retries
#   backlog_busy      — worker at its in-flight replay byte budget; client retries
StreamConnectionOutcome = Literal[
    "completed",
    "stream_error",
    "unavailable",
    "drained",
    "client_disconnect",
    "rotated",
    "backlog_error",
    "backlog_busy",
]
StreamWriteSkippedPath = Literal["ingest", "mirror", "relay"]
StreamTokenRoute = Literal["proxy", "django", "thin_tail_withheld"]
_ALLOWED_MODES = {"background", "interactive"}
_ALLOWED_RUN_SOURCES = {"manual", "signal_report", "agent"}
_ALLOWED_RUNTIME_ADAPTERS = {"claude", "codex"}
_ALLOWED_TASK_RUNTIMES = {"acp", "pi"}


TASK_RUN_CREATED_TOTAL = Counter(
    "posthog_tasks_task_run_created_total",
    "TaskRun rows created by the Tasks product",
    labelnames=[
        "origin_product",
        "run_environment",
        "mode",
        "run_source",
        "task_runtime",
        "runtime_adapter",
        "prewarmed",
    ],
)

TASK_RUN_STARTED_BESIDE_LIVE_RUN_TOTAL = Counter(
    "posthog_tasks_task_run_started_beside_live_run_total",
    "Cloud TaskRuns started while another cloud run of the same task was still live",
    labelnames=[
        "origin_product",
        "run_environment",
        "mode",
        "run_source",
        "task_runtime",
        "runtime_adapter",
        "prewarmed",
    ],
)

TASK_CONTRACT_VIOLATION_TOTAL = Counter(
    "posthog_tasks_contract_violation_total",
    "Task profile checks that a call failed. A profile that enforces the check refuses the call. Any other profile only counts it.",
    labelnames=["check", "profile"],
)

TASK_RUN_WORKFLOW_START_TOTAL = Counter(
    "posthog_tasks_task_run_workflow_start_total",
    "TaskRun workflow start lifecycle events",
    labelnames=[
        "origin_product",
        "run_environment",
        "mode",
        "run_source",
        "task_runtime",
        "runtime_adapter",
        "prewarmed",
        "outcome",
        "reason",
    ],
)

SANDBOX_RUN_START_TOTAL = Counter(
    "posthog_tasks_sandbox_run_start_total",
    "Run starts requested by task sandboxes, by endpoint and outcome",
    labelnames=["endpoint", "outcome"],
)

TASK_RUN_DISPATCH_CALLBACK_TOTAL = Counter(
    "posthog_tasks_task_run_dispatch_callback_total",
    "on_commit workflow-dispatch callback lifecycle: 'scheduled' when registered, 'fired' when it runs. "
    "scheduled minus fired is the count of lost callbacks that strand a run in QUEUED.",
    labelnames=[
        "origin_product",
        "run_environment",
        "mode",
        "run_source",
        "task_runtime",
        "runtime_adapter",
        "prewarmed",
        "phase",
    ],
)

WORKFLOW_DISPATCH_CREATED_TOTAL = Counter(
    "posthog_tasks_workflow_dispatch_created_total", "Workflow dispatch rows created", labelnames=["kind"]
)
WORKFLOW_DISPATCH_ATTEMPT_TOTAL = Counter(
    "posthog_tasks_workflow_dispatch_attempt_total",
    "Workflow dispatch attempt outcomes",
    labelnames=["kind", "outcome"],
)
WORKFLOW_DISPATCH_START_DURATION_SECONDS = Histogram(
    "posthog_tasks_workflow_dispatch_start_duration_seconds", "Temporal workflow start RPC duration"
)
WORKFLOW_DISPATCH_START_RPC_DURATION_SECONDS = Histogram(
    "posthog_tasks_workflow_dispatch_start_rpc_duration_seconds",
    "Temporal workflow start RPC duration",
    labelnames=["kind"],
    buckets=(0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 0.75, 1, 1.5, 2, 2.5, 3, 5, 7.5, 10),
)
WORKFLOW_DISPATCH_READY = Gauge("posthog_tasks_workflow_dispatch_ready", "Ready workflow dispatches")
WORKFLOW_DISPATCH_OLDEST_READY_AGE_SECONDS = Gauge(
    "posthog_tasks_workflow_dispatch_oldest_ready_age_seconds", "Age of the oldest ready workflow dispatch"
)
WORKFLOW_DISPATCH_CLAIMED = Gauge("posthog_tasks_workflow_dispatch_claimed", "Claimed workflow dispatches")
WORKFLOW_DISPATCH_LEASE_EXPIRED_TOTAL = Counter(
    "posthog_tasks_workflow_dispatch_lease_expired_total", "Expired workflow dispatch leases reclaimed"
)
WORKFLOW_DISPATCH_DEAD_TOTAL = Counter(
    "posthog_tasks_workflow_dispatch_dead_total", "Dead workflow dispatches", labelnames=["kind", "reason"]
)
WORKFLOW_DISPATCH_MISSING_INTENT_TOTAL = Counter(
    "posthog_tasks_workflow_dispatch_missing_intent_total", "Queued cloud task runs without dispatch intent"
)
SCHEDULED_TASK_RUN_MATERIALIZATION_TOTAL = Counter(
    "posthog_tasks_scheduled_task_run_materialization_total",
    "Scheduled task runs processed by the due-run materializer",
    labelnames=["outcome"],
)
SCHEDULED_TASK_RUN_DUE = Gauge(
    "posthog_tasks_scheduled_task_run_due",
    "Scheduled task runs whose requested start time has passed",
)
SCHEDULED_TASK_RUN_OLDEST_DUE_AGE_SECONDS = Gauge(
    "posthog_tasks_scheduled_task_run_oldest_due_age_seconds",
    "Age of the oldest due scheduled task run",
)

RUN_LOG_MIRROR_ENTRIES_TOTAL = Counter(
    "posthog_tasks_run_log_mirror_entries_total",
    "Task-run log entries mirrored to stdout for the internal Logs project (run_log_mirror).",
    labelnames=["origin_product"],
)

RUN_LOG_MIRROR_OTLP_BATCHES_TOTAL = Counter(
    "posthog_tasks_run_log_mirror_otlp_batches_total",
    "Direct-OTLP mirror batch deliveries by outcome (the local-dev leg; unset in cloud).",
    labelnames=["outcome"],
)

LOG_APPEND_UNSERIALIZED_TOTAL = Counter(
    "posthog_tasks_log_append_unserialized_total",
    "Task-run log appends that did not hold the per-object lock: refused on contention past the wait "
    "(the agent retries them) or run unserialized while redis is unavailable.",
    labelnames=["reason"],
)

PREWARMED_ACTIVATED_TOTAL = Counter(
    "posthog_tasks_prewarmed_activated_total",
    "Pre-warmed Runs that received their first user message (the warm sandbox got used, not reaped)",
    labelnames=["origin_product"],
)

PREWARMED_UNUSED_TOTAL = Counter(
    "posthog_tasks_prewarmed_unused_total",
    "Pre-warmed Runs that reached terminal without ever receiving a first user message — a sandbox was "
    "booted and paid for, then thrown away. The miss counterpart to posthog_tasks_prewarmed_activated_total: "
    "activated / (activated + unused) is the warm hit rate, and `reason` says where the misses go "
    "(a user who abandoned the composer looks different from a warm nobody released).",
    labelnames=["origin_product", "reason"],
)

TASK_RUN_FAILED_TOTAL = Counter(
    "posthog_tasks_task_run_failed_total",
    "TaskRun workflow failures with bounded attribution labels",
    labelnames=[
        "origin_product",
        "mode",
        "run_source",
        "runtime_adapter",
        "error_type",
        "temporal_activity_type",
        "temporal_activity_retry_state",
        "cause_error_type",
    ],
)

# Counted once per run on the terminal status write. TASK_RUN_FAILED_TOTAL only sees failures
# that raise, so it misses runs the workflow fails directly, such as an agent lost mid-turn.
TASK_RUN_TERMINAL_TOTAL = Counter(
    "posthog_tasks_task_run_terminal_total",
    "TaskRun terminal status transitions",
    labelnames=["origin_product", "run_environment", "status", "error_type", "termination_reason"],
)

CUSTOM_IMAGE_BUILD_TOTAL = Counter(
    "posthog_tasks_custom_image_build_total",
    "Custom sandbox image build lifecycle events",
    labelnames=["outcome"],
)

DEV_STACK_IMAGE_BAKE_TOTAL = Counter(
    "posthog_tasks_dev_stack_image_bake_total",
    "Prebaked dev-stack VM image bake lifecycle events",
    labelnames=["outcome", "region", "trigger"],
)

DEV_STACK_PREVIEW_BOOT_BUCKETS = [30.0, 60.0, 120.0, 180.0, 240.0, 300.0, 420.0, 600.0, 720.0]

DEV_STACK_PREVIEW_BOOT_SECONDS = Histogram(
    "posthog_tasks_dev_stack_preview_boot_seconds",
    "Wall time from launching the dev stack in a run's sandbox to its preview answering health checks",
    buckets=DEV_STACK_PREVIEW_BOOT_BUCKETS,
)

DEV_STACK_PREVIEW_TOTAL = Counter(
    "posthog_tasks_dev_stack_preview_total",
    "Dev stack preview lifecycle outcomes. started counts launches and attached counts re-entries that joined "
    "a launch in progress; ready, failed, timed_out and cancelled each close one of those; launch_failed "
    "counts start attempts that raised before launching.",
    labelnames=["outcome"],
)


# Connection lifetimes range from a few seconds (cold reconnect) to the
# per-connection cap. The 120s bucket isolates connections cut at the
# Envoy/Contour response_timeout boundary from genuinely long-lived ones.
STREAM_CONNECTION_DURATION_BUCKETS = [
    1.0,
    5.0,
    15.0,
    30.0,
    60.0,
    120.0,
    300.0,
    600.0,
    960.0,
    1_800.0,
    3_600.0,
    7_200.0,
    21_600.0,
]
# Stream length is capped at TASK_RUN_STREAM_MAX_LENGTH (~5k); the top buckets
# show how close real runs get to the trim threshold.
STREAM_LENGTH_BUCKETS = [10.0, 50.0, 100.0, 500.0, 1_000.0, 2_500.0, 5_000.0, 10_000.0, 15_000.0, 20_000.0]


TASK_RUN_STREAM_CONNECTIONS_OPENED_TOTAL = Counter(
    "posthog_tasks_task_run_stream_connections_opened_total",
    "SSE task-run stream connections opened",
    labelnames=["origin_product"],
)

TASK_RUN_STREAM_BACKLOG_ENTRIES_TOTAL = Counter(
    "posthog_tasks_task_run_stream_backlog_entries_total",
    "Run-log entries served as connect-time backlog on thin-tail SSE stream connections",
    labelnames=["origin_product"],
)

TASK_RUN_STREAM_BACKLOG_GAP_TOTAL = Counter(
    "posthog_tasks_task_run_stream_backlog_gap_total",
    "Thin-tail SSE connections whose first uncovered live entry was not contiguous with the log backlog",
    labelnames=["origin_product"],
)

STREAM_BACKLOG_BYTES_BUCKETS = [
    65_536.0,
    262_144.0,
    1_048_576.0,
    4_194_304.0,
    16_777_216.0,
    52_428_800.0,
    134_217_728.0,
    536_870_912.0,
]

TASK_RUN_STREAM_BACKLOG_BYTES = Histogram(
    "posthog_tasks_task_run_stream_backlog_bytes",
    "Run-log byte size measured before a connect-time backlog replay on thin-tail SSE stream connections",
    labelnames=["origin_product"],
    buckets=STREAM_BACKLOG_BYTES_BUCKETS,
)

TASK_RUN_STREAM_BACKLOG_OVERSIZED_TOTAL = Counter(
    "posthog_tasks_task_run_stream_backlog_oversized_total",
    "Thin-tail SSE connections whose run log exceeded the backlog byte cap and degraded to the Redis window",
    labelnames=["origin_product"],
)

TASK_RUN_STREAM_BACKLOG_THROTTLED_TOTAL = Counter(
    "posthog_tasks_task_run_stream_backlog_throttled_total",
    "Thin-tail SSE connections refused a backlog replay because the worker's in-flight replay byte budget was full",
    labelnames=["origin_product"],
)

TASK_RUN_STREAM_CONNECTIONS_CLOSED_TOTAL = Counter(
    "posthog_tasks_task_run_stream_connections_closed_total",
    "SSE task-run stream connections closed, labeled by how they ended",
    labelnames=["origin_product", "outcome"],
)

TASK_RUN_STREAM_CONNECTION_DURATION_SECONDS = Histogram(
    "posthog_tasks_task_run_stream_connection_duration_seconds",
    "Lifetime of an SSE task-run stream connection",
    labelnames=["origin_product", "outcome"],
    buckets=STREAM_CONNECTION_DURATION_BUCKETS,
)

TASK_RUN_STREAM_LENGTH_ON_CONNECT = Histogram(
    "posthog_tasks_task_run_stream_length_on_connect",
    "Redis stream length observed when an SSE connection reconnects with a cursor",
    buckets=STREAM_LENGTH_BUCKETS,
)

TASK_RUN_STREAM_RESUME_GAP_TOTAL = Counter(
    "posthog_tasks_task_run_stream_resume_gap_total",
    "SSE reconnects whose Last-Event-ID was already trimmed from Redis (events lost for that client)",
    labelnames=["origin_product"],
)

TASK_RUN_STREAM_TOKEN_ROUTED_TOTAL = Counter(
    "posthog_tasks_task_run_stream_token_routed_total",
    "Stream read tokens minted, labeled by the read leg the client was routed to and whether it asked to resync",
    labelnames=["origin_product", "route", "resync_requested"],
)

TASK_RUN_STREAM_WRITE_SKIPPED_TOTAL = Counter(
    "posthog_tasks_task_run_stream_write_skipped_total",
    "Task-run events not mirrored into Redis because presence gating found no attached reader",
    labelnames=["path", "origin_product"],
)

TASK_RUN_SANDBOX_PROCESS_KILLED_TOTAL = Counter(
    "posthog_tasks_sandbox_process_killed_notifications_total",
    "Sandbox memory watchdog kills captured from the sequenced event ingest",
)

TASK_RUN_AGENT_FAILURE_TOTAL = Counter(
    "posthog_tasks_agent_turn_failed_total",
    "TaskRun transitions to FAILED via the API facade (agent-server turn failures)",
    labelnames=["origin_product", "mode", "run_source", "task_runtime", "runtime_adapter"],
)

TASK_RUN_FOLLOWUP_DELIVERY_FAILED_TOTAL = Counter(
    "posthog_tasks_followup_delivery_failed_total",
    "Follow-up user message deliveries to a live sandbox that failed",
    labelnames=["origin_product", "retryable"],
)

ACTIVE_SESSION_CAP_HITS_TOTAL = Counter(
    "posthog_tasks_active_session_cap_hits_total",
    "Run starts and warm claims refused because the user or organization already holds the max open cloud sandbox sessions",
    labelnames=["origin_product", "scope"],
)

WARM_CHURN_THROTTLED_TOTAL = Counter(
    "posthog_tasks_warm_churn_throttled_total",
    "Warm requests refused because the user ended too many unclaimed warm sandboxes in the last hour",
    labelnames=["origin_product"],
)

SANDBOX_DEADLINE_BUCKETS = [5.0, 15.0, 30.0, 60.0, 120.0, 180.0, 300.0, 600.0]

SANDBOX_DEADLINE_TOTAL = Counter(
    "posthog_tasks_sandbox_deadline_total",
    "Interactive cloud runs that reached the pre-deadline lead time on their sandbox, "
    "by what the run ended up on. rotated moved onto a replacement sandbox; snapshot_only "
    "kept the sandbox it had and saved the session; snapshot_failed kept it with nothing "
    "saved; routing_lost backed out of a rotation and could not repoint the run, so it can "
    "take no more messages.",
    labelnames=["outcome", "reason", "origin_product"],
)

SANDBOX_ROTATION_DURATION_SECONDS = Histogram(
    "posthog_tasks_sandbox_rotation_duration_seconds",
    "Wall time a run spent handling its sandbox deadline. The run answers no messages for "
    "this long, so it is the user-visible cost of a rotation.",
    labelnames=["outcome"],
    buckets=SANDBOX_DEADLINE_BUCKETS,
)

FOLLOWUP_SANDBOX_STOPPED_TOTAL = Counter(
    "posthog_tasks_followup_sandbox_stopped_total",
    "Follow-up deliveries rejected because the control plane reports the run's sandbox "
    "stopped, by where the check caught it",
    labelnames=["origin_product", "detected_by"],
)

FOLLOWUP_DENIED_PERMISSION_STOP_TOTAL = Counter(
    "posthog_tasks_followup_denied_permission_stop_total",
    "Follow-up deliveries dropped instead of retried because the turn ended on a permission "
    "the actor denied. A retry would re-ask the refused question.",
    labelnames=["origin_product"],
)

TASK_RUN_WIZARD_UNBOUND_TOTAL = Counter(
    "posthog_tasks_wizard_run_unbound_total",
    "Wizard cloud runs that reached a terminal status without an output.pr_url binding",
    labelnames=["status"],
)

TURN_COMPLETED_SUPPRESSED_TOTAL = Counter(
    "posthog_tasks_turn_completed_suppressed_total",
    "Interactive turn completion notifications and activity updates suppressed by ingest",
    labelnames=["reason"],
)

PUSH_DISPATCHER_FAILURES_TOTAL = Counter(
    "posthog_tasks_push_dispatcher_failures_total",
    "Push-notification dispatch attempts that failed and were swallowed by the best-effort dispatcher",
    labelnames=["kind", "reason"],
)

PUSH_DISPATCHER_OUTCOMES_TOTAL = Counter(
    "posthog_tasks_push_dispatcher_outcomes_total",
    "Push-notification dispatcher decisions before the Celery delivery task",
    labelnames=["kind", "outcome"],
)

# reason is one of: created, replayed, gate_blocked, rate_capped, team_rate_capped,
# limit_reached, owner_ineligible, a fixed, code-defined set, safe as a label.
WORKFLOW_TASK_CREATE_TOTAL = Counter(
    "posthog_tasks_workflow_task_create_total",
    'Workflow "Create AI task" action outcomes',
    labelnames=["reason"],
)

CodeUsageGateOutcome = Literal["checked_allowed", "checked_blocked", "fail_open", "org_deactivated"]
ComputeQuotaOutcome = Literal["checked_allowed", "checked_blocked", "fail_open"]
DesktopAccessOutcome = Literal[
    "allowed",
    "startup_plan",
    "prepaid_credits",
    "signups_paused",
    "override",
    "resolution_failure",
]

# outcome: checked_allowed/checked_blocked when the LLM gateway answered the usage check,
# fail_open when a gateway/token error let the run proceed unchecked (a degraded gateway must not silently remove the only cost backstop), org_deactivated when
# the local deactivated-organization check blocked the run before any gateway call.
CODE_USAGE_GATE_CHECK_TOTAL = Counter(
    "posthog_tasks_code_usage_gate_check_total",
    "Cloud usage-gate check outcomes for PostHog Code runs",
    labelnames=["outcome"],
)

COMPUTE_QUOTA_CHECK_TOTAL = Counter(
    "posthog_tasks_compute_quota_check_total",
    "Compute quota-check outcomes for billable PostHog Desktop runs",
    labelnames=["outcome"],
)

DESKTOP_ACCESS_DECISIONS_TOTAL = Counter(
    "posthog_tasks_desktop_access_decisions_total",
    "PostHog Desktop access decisions by bounded outcome",
    labelnames=["outcome"],
)

DESKTOP_ACCESS_FLAG_UNRESOLVED_TOTAL = Counter(
    "posthog_tasks_desktop_access_flag_unresolved_total",
    "PostHog Desktop access flag evaluations that returned no boolean value",
    labelnames=["flag"],
)


def observe_compute_quota_check(outcome: ComputeQuotaOutcome) -> None:
    COMPUTE_QUOTA_CHECK_TOTAL.labels(outcome=outcome).inc()


ComputeSettlementOutcome = Literal[
    "settled",
    "duplicate",
    "settled_locally",
    "conflict",
    "retry",
    "attempts_exhausted",
    "contract_error",
    "insufficient_credits",
    "deployment_error",
    "skipped",
]

COMPUTE_SETTLEMENTS_TOTAL = Counter(
    "posthog_tasks_compute_settlements_total",
    "Sandbox compute settlement attempts against the gateway ledger, by outcome",
    labelnames=["outcome"],
)


def observe_compute_settlement(outcome: ComputeSettlementOutcome) -> None:
    COMPUTE_SETTLEMENTS_TOTAL.labels(outcome=outcome).inc()


def push_compute_settlement_sweep_gauges(
    *, hogland_open_over_day: int, unsettled_over_hour: int, parked_by_reason: Mapping[str, int]
) -> None:
    with pushed_metrics_registry("tasks_compute_settlement") as registry:
        Gauge(
            "posthog_tasks_hogland_sandbox_sessions_open_over_day",
            "Hogland sandbox sessions with no end stamp that opened more than a day ago",
            registry=registry,
        ).set(hogland_open_over_day)
        Gauge(
            "posthog_tasks_sandbox_sessions_unsettled_over_hour",
            "Attributed sandbox sessions that ended more than an hour ago and are neither settled nor parked",
            registry=registry,
        ).set(unsettled_over_hour)
        parked = Gauge(
            "posthog_tasks_sandbox_sessions_parked",
            "Sandbox sessions that are parked and not settled, by park reason",
            labelnames=["reason"],
            registry=registry,
        )
        for reason, rows in parked_by_reason.items():
            parked.labels(reason=reason).set(rows)


# scoped: "true" when the delivery's installation resolved to at least one team, so the
# TaskRun lookup could ride the team_id index. "false" means no team links the installation,
# so the lookup is skipped and the delivery matches no run.
GITHUB_WEBHOOK_TASK_RUN_LOOKUP_TOTAL = Counter(
    "posthog_tasks_github_webhook_task_run_lookup_total",
    "GitHub webhook TaskRun lookups, labeled by whether they were scoped to the installation's teams",
    labelnames=["scoped"],
)


def observe_github_webhook_task_run_lookup(*, scoped: bool) -> None:
    GITHUB_WEBHOOK_TASK_RUN_LOOKUP_TOTAL.labels(scoped="true" if scoped else "false").inc()


MCP_EXCLUDE_TOOLS_OVER_CAP_TOTAL = Counter(
    "posthog_tasks_mcp_exclude_tools_over_cap_total",
    "Profiled runs refused because the profile and the run together hide more MCP tools than the sandbox cap",
    labelnames=["profile"],
)


def observe_mcp_exclude_tools_over_cap(*, profile: str) -> None:
    MCP_EXCLUDE_TOOLS_OVER_CAP_TOTAL.labels(profile=profile).inc()


def _metric_label(value: object | None) -> str:
    if value is None:
        return "unknown"
    if hasattr(value, "value"):
        return str(value.value)
    return str(value)


def _bounded_metric_label(value: object | None, allowed_values: set[str]) -> str:
    label = _metric_label(value)
    if label == "unknown" or label in allowed_values:
        return label
    return "other"


def _failure_metric_label(value: object | None) -> str:
    label = _metric_label(value)
    if label == "unknown":
        return label
    return label[:100]


def _task_runtime_label(task_run: "TaskRun | None") -> str:
    if task_run is None:
        return "unknown"
    return _bounded_metric_label(getattr(task_run.task, "runtime", None), _ALLOWED_TASK_RUNTIMES)


def _effective_runtime_adapter_label(task_run: "TaskRun | None", state: dict) -> str:
    task_runtime = _task_runtime_label(task_run)
    if task_runtime == "pi":
        return "pi"

    configured_adapter = state.get("runtime_adapter")
    if configured_adapter is not None:
        return _bounded_metric_label(configured_adapter, _ALLOWED_RUNTIME_ADAPTERS)

    # ACP's default harness is Claude. A model-only override deliberately leaves
    # runtime_adapter unset, so treating that valid configuration as unknown hides
    # the runtime that actually handled the run.
    return "claude" if task_runtime == "acp" else "unknown"


def _task_run_labels(task_run: "TaskRun | None") -> dict[str, str]:
    if task_run is None:
        return {
            "origin_product": "unknown",
            "run_environment": "unknown",
            "mode": "unknown",
            "run_source": "unknown",
            "task_runtime": "unknown",
            "runtime_adapter": "unknown",
            "prewarmed": "unknown",
        }

    state = task_run.state if isinstance(task_run.state, dict) else {}
    return {
        "origin_product": origin_product_label(task_run),
        "run_environment": _metric_label(task_run.environment),
        "mode": _bounded_metric_label(state.get("mode"), _ALLOWED_MODES),
        "run_source": _bounded_metric_label(state.get("run_source"), _ALLOWED_RUN_SOURCES),
        "task_runtime": _task_runtime_label(task_run),
        "runtime_adapter": _effective_runtime_adapter_label(task_run, state),
        "prewarmed": "true" if state.get("prewarmed") else "false",
    }


def observe_task_run_created(task_run: "TaskRun") -> None:
    TASK_RUN_CREATED_TOTAL.labels(**_task_run_labels(task_run)).inc()


def observe_task_run_started_beside_live_run(task_run: "TaskRun") -> None:
    TASK_RUN_STARTED_BESIDE_LIVE_RUN_TOTAL.labels(**_task_run_labels(task_run)).inc()


def observe_contract_violation(*, check: str, profile: str) -> None:
    TASK_CONTRACT_VIOLATION_TOTAL.labels(check=check, profile=profile).inc()


def observe_sandbox_run_start(*, endpoint: str, outcome: str) -> None:
    SANDBOX_RUN_START_TOTAL.labels(endpoint=endpoint, outcome=outcome).inc()


def observe_task_run_dispatch_callback(task_run: "TaskRun | None", *, phase: Literal["scheduled", "fired"]) -> None:
    TASK_RUN_DISPATCH_CALLBACK_TOTAL.labels(**_task_run_labels(task_run), phase=phase).inc()


def observe_task_run_workflow_start(
    task_run: "TaskRun | None",
    *,
    outcome: TaskWorkflowStartOutcome,
    reason: str,
) -> None:
    TASK_RUN_WORKFLOW_START_TOTAL.labels(
        **_task_run_labels(task_run),
        outcome=outcome,
        reason=reason,
    ).inc()


def observe_prewarmed_activated(task_run: "TaskRun") -> None:
    PREWARMED_ACTIVATED_TOTAL.labels(origin_product=origin_product_label(task_run)).inc()


def observe_prewarmed_unused(task_run: "TaskRun", *, reason: PrewarmedUnusedReason) -> None:
    """Count a warm Run that terminalized still awaiting its first message.

    Callers must check `state.prewarmed` and `state.await_user_message` first — activation clears the
    latter, so a run that still carries it never got used. Never raises: a metric must not fail a
    terminal status transition.
    """
    try:
        PREWARMED_UNUSED_TOTAL.labels(origin_product=origin_product_label(task_run), reason=reason).inc()
    except Exception:
        logger.exception("prewarmed_unused_metric_failed", run_id=str(task_run.id))


def observe_prewarmed_unused_if_never_activated(task_run: "TaskRun", *, reason: PrewarmedUnusedReason) -> None:
    """Count a terminal Run as an unused warm, if that is what it is.

    Owns the "never used" test so every path that terminalizes a Run books the miss the same way —
    the workflow's status activity and the direct status write the cancel fallback uses when the
    workflow is already gone.

    `warm_activated` is what separates a real miss from a race: activation sets that marker before it
    signals the first message and clears `await_user_message` only after, so a Run that terminalizes
    between the two still carries both of the older markers while already being counted as activated.
    """
    run_state = task_run.state if isinstance(task_run.state, dict) else {}
    if not run_state.get("prewarmed") or not run_state.get("await_user_message"):
        return
    if run_state.get("warm_activated"):
        return
    observe_prewarmed_unused(task_run, reason=reason)


def observe_custom_image_build(outcome: CustomImageBuildOutcome) -> None:
    try:
        CUSTOM_IMAGE_BUILD_TOTAL.labels(outcome=outcome).inc()
    except Exception:
        logger.exception("custom_image_build_metric_failed", outcome=outcome)


def observe_dev_stack_image_bake(outcome: DevStackImageBakeOutcome, *, trigger: str) -> None:
    try:
        from posthog.utils import get_instance_region  # noqa: PLC0415

        DEV_STACK_IMAGE_BAKE_TOTAL.labels(
            outcome=outcome,
            region=get_instance_region() or "unknown",
            trigger=trigger,
        ).inc()
    except Exception:
        logger.exception("dev_stack_image_bake_metric_failed", outcome=outcome, trigger=trigger)


def origin_product_label(task_run: "TaskRun | None") -> str:
    """Bounded origin_product metric label resolved from the task run's task."""
    if task_run is None:
        return "unknown"
    return _metric_label(getattr(task_run.task, "origin_product", None))


def observe_stream_connection_opened(origin_product: str) -> None:
    TASK_RUN_STREAM_CONNECTIONS_OPENED_TOTAL.labels(origin_product=origin_product).inc()


def observe_stream_backlog_served(origin_product: str, entries: int) -> None:
    if entries > 0:
        TASK_RUN_STREAM_BACKLOG_ENTRIES_TOTAL.labels(origin_product=origin_product).inc(entries)


def observe_stream_backlog_gap(origin_product: str) -> None:
    TASK_RUN_STREAM_BACKLOG_GAP_TOTAL.labels(origin_product=origin_product).inc()


def observe_stream_backlog_bytes(origin_product: str, size_bytes: int) -> None:
    TASK_RUN_STREAM_BACKLOG_BYTES.labels(origin_product=origin_product).observe(size_bytes)


def observe_stream_backlog_oversized(origin_product: str) -> None:
    TASK_RUN_STREAM_BACKLOG_OVERSIZED_TOTAL.labels(origin_product=origin_product).inc()


def observe_stream_backlog_throttled(origin_product: str) -> None:
    TASK_RUN_STREAM_BACKLOG_THROTTLED_TOTAL.labels(origin_product=origin_product).inc()


def observe_stream_connection_closed(
    origin_product: str, outcome: StreamConnectionOutcome, duration_seconds: float
) -> None:
    TASK_RUN_STREAM_CONNECTIONS_CLOSED_TOTAL.labels(origin_product=origin_product, outcome=outcome).inc()
    TASK_RUN_STREAM_CONNECTION_DURATION_SECONDS.labels(origin_product=origin_product, outcome=outcome).observe(
        duration_seconds
    )


def observe_stream_length_on_connect(length: int) -> None:
    TASK_RUN_STREAM_LENGTH_ON_CONNECT.observe(length)


def observe_stream_resume_gap(origin_product: str) -> None:
    TASK_RUN_STREAM_RESUME_GAP_TOTAL.labels(origin_product=origin_product).inc()


def observe_stream_token_routed(origin_product: str, route: StreamTokenRoute, resync_requested: bool) -> None:
    TASK_RUN_STREAM_TOKEN_ROUTED_TOTAL.labels(
        origin_product=_metric_label(origin_product),
        route=route,
        resync_requested="true" if resync_requested else "false",
    ).inc()


def observe_stream_write_skipped(path: StreamWriteSkippedPath, origin_product: str | None = None) -> None:
    TASK_RUN_STREAM_WRITE_SKIPPED_TOTAL.labels(path=path, origin_product=_metric_label(origin_product)).inc()


def observe_sandbox_process_killed() -> None:
    TASK_RUN_SANDBOX_PROCESS_KILLED_TOTAL.inc()


def observe_task_run_failed(properties: dict[str, object]) -> None:
    TASK_RUN_FAILED_TOTAL.labels(
        origin_product=_metric_label(properties.get("origin_product")),
        mode=_bounded_metric_label(properties.get("mode"), _ALLOWED_MODES),
        run_source=_bounded_metric_label(properties.get("run_source"), _ALLOWED_RUN_SOURCES),
        runtime_adapter=_bounded_metric_label(properties.get("runtime_adapter"), _ALLOWED_RUNTIME_ADAPTERS),
        error_type=_failure_metric_label(properties.get("error_type")),
        temporal_activity_type=_failure_metric_label(properties.get("temporal_activity_type")),
        temporal_activity_retry_state=_failure_metric_label(properties.get("temporal_activity_retry_state")),
        cause_error_type=_failure_metric_label(properties.get("cause_error_type")),
    ).inc()


def observe_agent_turn_failed(task_run: "TaskRun") -> None:
    labels = _task_run_labels(task_run)
    TASK_RUN_AGENT_FAILURE_TOTAL.labels(
        origin_product=labels["origin_product"],
        mode=labels["mode"],
        run_source=labels["run_source"],
        task_runtime=labels["task_runtime"],
        runtime_adapter=labels["runtime_adapter"],
    ).inc()


_WIZARD_TERMINAL_STATUSES = {"completed", "failed", "cancelled"}


def observe_task_run_terminal(task_run: "TaskRun", *, error_type: str | None, termination_reason: str | None) -> None:
    labels = _task_run_labels(task_run)
    TASK_RUN_TERMINAL_TOTAL.labels(
        origin_product=labels["origin_product"],
        run_environment=labels["run_environment"],
        status=_metric_label(task_run.status),
        error_type=_failure_metric_label(error_type),
        termination_reason=_metric_label(termination_reason),
    ).inc()


def observe_wizard_run_unbound(task_run: "TaskRun") -> None:
    """Record a wizard run ending without its PR ever binding.

    Safe to call on any status write; only terminal wizard runs without an
    output.pr_url count. Every binding failure mode is silent (agent used a
    different branch, webhook undelivered, write swallowed), so this counter
    is the only signal that the wizard PR pipeline regressed.
    """
    if task_run.status not in _WIZARD_TERMINAL_STATUSES:
        return
    state = task_run.state if isinstance(task_run.state, dict) else {}
    if not state.get("wizard_head_branch"):
        return
    output = task_run.output if isinstance(task_run.output, dict) else {}
    if output.get("pr_url"):
        return
    TASK_RUN_WIZARD_UNBOUND_TOTAL.labels(status=task_run.status).inc()
    logger.warning(
        "wizard_run_terminal_without_pr",
        run_id=str(task_run.id),
        status=task_run.status,
        wizard_head_branch=state.get("wizard_head_branch"),
    )


def observe_followup_delivery_failed(task_run: "TaskRun", *, retryable: bool) -> None:
    TASK_RUN_FOLLOWUP_DELIVERY_FAILED_TOTAL.labels(
        origin_product=origin_product_label(task_run),
        retryable="true" if retryable else "false",
    ).inc()


def observe_active_session_cap_hit(origin_product: str, scope: str) -> None:
    ACTIVE_SESSION_CAP_HITS_TOTAL.labels(origin_product=_metric_label(origin_product), scope=scope).inc()


def observe_warm_churn_throttled(origin_product: str) -> None:
    WARM_CHURN_THROTTLED_TOTAL.labels(origin_product=_metric_label(origin_product)).inc()


_SANDBOX_DEADLINE_OUTCOMES = {"rotated", "snapshot_only", "snapshot_failed", "routing_lost"}
_SANDBOX_DEADLINE_REASONS = {
    "none",
    "no_sandbox",
    "flag_disabled",
    "agent_active",
    "turn_open",
    "followup_in_flight",
    "run_completed",
    "snapshot_missing",
    "provision_failed",
    "snapshot_unused",
    "agent_activity",
    "activity_during_snapshot",
    "activity_during_handoff",
}


def observe_sandbox_deadline(properties: dict[str, object]) -> None:
    outcome = _bounded_metric_label(properties.get("outcome"), _SANDBOX_DEADLINE_OUTCOMES)
    SANDBOX_DEADLINE_TOTAL.labels(
        outcome=outcome,
        reason=_bounded_metric_label(properties.get("reason"), _SANDBOX_DEADLINE_REASONS),
        origin_product=_metric_label(properties.get("origin_product")),
    ).inc()
    duration = properties.get("duration_seconds")
    if isinstance(duration, int | float) and not isinstance(duration, bool):
        SANDBOX_ROTATION_DURATION_SECONDS.labels(outcome=outcome).observe(float(duration))


def observe_followup_sandbox_stopped(task_run: "TaskRun | None", *, detected_by: str) -> None:
    FOLLOWUP_SANDBOX_STOPPED_TOTAL.labels(
        origin_product=origin_product_label(task_run),
        detected_by=detected_by,
    ).inc()


def observe_followup_denied_permission_stop(task_run: "TaskRun | None") -> None:
    FOLLOWUP_DENIED_PERMISSION_STOP_TOTAL.labels(origin_product=origin_product_label(task_run)).inc()


def observe_workflow_task_create(*, reason: str) -> None:
    WORKFLOW_TASK_CREATE_TOTAL.labels(reason=reason).inc()


def observe_code_usage_gate_check(*, outcome: CodeUsageGateOutcome) -> None:
    CODE_USAGE_GATE_CHECK_TOTAL.labels(outcome=outcome).inc()


def observe_desktop_access_decision(*, outcome: DesktopAccessOutcome) -> None:
    DESKTOP_ACCESS_DECISIONS_TOTAL.labels(outcome=outcome).inc()


def observe_desktop_access_flag_unresolved(*, flag: str) -> None:
    DESKTOP_ACCESS_FLAG_UNRESOLVED_TOTAL.labels(flag=flag).inc()