Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

10.9 KB
Raw Blame
"""Node-run observability: one recorder for every terminal run transition.

Every path that moves a NotebookNodeRun to a terminal status reports it here exactly once:
the sandbox callback, the direct lane, dispatch failures, and interrupts. The recorder
measures end-to-end duration against the run row's ``created_at``
(the only start clock that exists) and emits three sinks at once:

- a Prometheus histogram (Grafana / the rollout dashboard),
- its OTLP twin (the PostHog Metrics product),
- a ``notebook node run completed`` event (product analytics, sliceable per notebook).

Every sink also carries ``delivery``: which transport served the run's rows
(``direct`` / ``inline`` / ``object_relay`` / ``object_ch_writes`` / ``none`` / ``mixed``).
The data plane resolves it when it picks a transport, so it reflects what actually
happened rather than what the flags asked for, which makes a fallback visible instead of
silently landing in the wrong bucket. This is the dimension a transport A/B splits on.

Runs additionally carry phase timings in the envelope's ``timings`` dict. Kernel runs
report them from the sandbox (``sandbox/kernel/runner.py``): ``input_wait_s`` (waiting on
the data plane for referenced frames or the display fetch), ``download_s`` (the presigned
frame downloads), ``kernel_boot_s`` (ensuring the ipykernel is up — ~0 when already warm),
``exec_s`` (ipykernel cell execution), and ``sandbox_total_s`` (the whole
sandbox-side run). Direct (hogql) runs report them from the async query manager's status
(``sql_v2_direct._query_status_timings``): ``queued_s`` (enqueue -> Celery pickup) and
``clickhouse_s`` (pickup -> completion, i.e. HogQL compile + ClickHouse execution).
"""

import math
from typing import Any

from django.core.exceptions import ObjectDoesNotExist
from django.utils import timezone

import structlog
from prometheus_client import Counter, Histogram

from posthog.event_usage import report_user_or_team_action
from posthog.models import Team
from posthog.otel_metrics import OtelInstrumentFactory

from products.notebooks.backend.models import NotebookNodeRun
from products.notebooks.backend.sql_v2 import (
    DELIVERY_DIRECT,
    DELIVERY_INLINE,
    DELIVERY_MIXED,
    DELIVERY_NONE,
    DELIVERY_OBJECT_CH_WRITES,
    DELIVERY_OBJECT_RELAY,
)

logger = structlog.get_logger(__name__)

NODE_RUN_COMPLETED_EVENT = "notebook node run completed"
NOTEBOOK_RUN_COMPLETED_EVENT = "notebook run completed"

# The terminal statuses, plus `timed_out` — a FAILED written by the direct lane's
# grace-expiry watchdog. Kept distinct so an expired query is a visible bucket rather
# than indistinguishable from a user error.
OUTCOME_DONE = "done"
OUTCOME_FAILED = "failed"
OUTCOME_INTERRUPTED = "interrupted"
OUTCOME_TIMED_OUT = "timed_out"

_OUTCOME_BY_STATUS = {
    NotebookNodeRun.Status.DONE: OUTCOME_DONE,
    NotebookNodeRun.Status.FAILED: OUTCOME_FAILED,
    NotebookNodeRun.Status.INTERRUPTED: OUTCOME_INTERRUPTED,
}

# Envelope timing keys -> phase label values. Anything else in `timings` is ignored, so a
# hostile or newer sandbox cannot mint unbounded label values.
_PHASE_BY_TIMING_KEY = {
    "input_wait_s": "input_wait",
    "download_s": "download",
    "kernel_boot_s": "kernel_boot",
    "exec_s": "exec",
    "sandbox_total_s": "sandbox_total",
    "queued_s": "queued",
    "clickhouse_s": "clickhouse",
}

# Headroom over the backend-measured run duration when bounding sandbox-reported timings:
# absorbs clock granularity and rounding without letting a forged value stretch far past
# the run's real wall clock.
_TIMING_CLOCK_SLACK_SECONDS = 60.0

# The envelope's `delivery` reaches us through the sandbox, where user code can forge it, so
# only these values may become a label. Anything else collapses to DELIVERY_NONE rather than
# minting a new time series per forged string. Keep in sync with the DELIVERY_* constants in
# sql_v2.py when a transport is added.
DELIVERY_LABEL_VALUES = frozenset(
    {
        DELIVERY_DIRECT,
        DELIVERY_INLINE,
        DELIVERY_OBJECT_RELAY,
        DELIVERY_OBJECT_CH_WRITES,
        DELIVERY_NONE,
        DELIVERY_MIXED,
    }
)

_otel = OtelInstrumentFactory("notebooks")

NODE_RUN_SECONDS = Histogram(
    "posthog_notebooks_node_run_seconds",
    "End-to-end notebook node run duration: run-row creation to its terminal transition.",
    labelnames=["node_type", "outcome", "delivery"],
    buckets=[0.5, 1, 2, 5, 10, 30, 60, 120, 300, 600, 1200, 1800, 2700],
)
NODE_RUN_PHASE_SECONDS = Histogram(
    "posthog_notebooks_node_run_phase_seconds",
    "Phase durations of a node run, from the run envelope's timings.",
    labelnames=["phase", "node_type"],
    buckets=[0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60, 120, 300, 600],
)


NOTEBOOK_RUN_TERMINAL = Counter(
    "posthog_notebooks_notebook_run_terminal_total",
    "Whole-notebook runs that reached a terminal state, by outcome and by the surface that started them.",
    labelnames=["outcome", "trigger"],
)
NOTEBOOK_RUN_SECONDS = Histogram(
    "posthog_notebooks_notebook_run_seconds",
    "End-to-end whole-notebook run duration: run-row creation to its terminal transition.",
    labelnames=["outcome", "trigger"],
    buckets=[1, 5, 10, 30, 60, 120, 300, 600, 1200, 1800, 3600],
)


def outcome_for_status(status: str) -> str:
    return _OUTCOME_BY_STATUS.get(NotebookNodeRun.Status(status), OUTCOME_FAILED)


def record_notebook_run_terminal(notebook_run: Any, outcome: str) -> None:
    """Report one terminal transition of a whole-notebook run. Call only when this caller
    won the transition, so a retried activity never double-counts.

    Never raises — telemetry must not fail the run.
    """
    try:
        duration = max((timezone.now() - notebook_run.created_at).total_seconds(), 0.0)
        labels = {"outcome": outcome, "trigger": notebook_run.trigger}
        NOTEBOOK_RUN_TERMINAL.labels(**labels).inc()
        NOTEBOOK_RUN_SECONDS.labels(**labels).observe(duration)
        _otel.record_histogram_twin(NOTEBOOK_RUN_SECONDS, duration, labels)

        cell_plan = notebook_run.cell_plan if isinstance(notebook_run.cell_plan, list) else []
        properties: dict[str, Any] = {
            "notebook_short_id": notebook_run.notebook.short_id,
            "trigger": notebook_run.trigger,
            "outcome": outcome,
            "cell_count": len(cell_plan),
            "python_cell_count": sum(1 for cell in cell_plan if cell.get("cell_type") == "python"),
            "completed_count": notebook_run.current_index,
            "duration_ms": round(duration * 1000),
            "has_error": bool(notebook_run.error),
        }
        try:
            user = notebook_run.user
        except ObjectDoesNotExist:
            # The FK is db_constraint=False/DO_NOTHING, the same as NotebookNodeRun.user.
            user = None
        team = Team.objects.filter(pk=notebook_run.team_id).first()
        report_user_or_team_action(NOTEBOOK_RUN_COMPLETED_EVENT, properties, user=user, team=team)
    except Exception:
        logger.exception("notebook_run_metrics_failed", notebook_run_id=str(notebook_run.id), outcome=outcome)


def record_node_run_terminal(run: NotebookNodeRun, outcome: str) -> None:
    """Report one terminal transition of `run`. Call only when this caller won the
    RUNNING -> terminal transition, so re-deliveries and racing pollers never double-count.

    Never raises — telemetry must not fail the run.
    """
    try:
        duration = max((timezone.now() - run.created_at).total_seconds(), 0.0)
        delivery = _sanitized_delivery(run.envelope)
        run_labels = {"node_type": run.node_type, "outcome": outcome, "delivery": delivery}
        NODE_RUN_SECONDS.labels(**run_labels).observe(duration)
        _otel.record_histogram_twin(NODE_RUN_SECONDS, duration, run_labels)

        timings = _sanitized_timings(run.envelope, max_seconds=duration + _TIMING_CLOCK_SLACK_SECONDS)
        for phase, seconds in timings.items():
            NODE_RUN_PHASE_SECONDS.labels(phase=phase, node_type=run.node_type).observe(seconds)
            _otel.record_histogram_twin(NODE_RUN_PHASE_SECONDS, seconds, {"phase": phase, "node_type": run.node_type})

        _capture_completed_event(run, outcome, duration, timings, delivery)
    except Exception:
        logger.exception("notebook_node_run_metrics_failed", run_id=str(run.id), outcome=outcome)


def _sanitized_timings(envelope: Any, max_seconds: float) -> dict[str, float]:
    """Bound the envelope's self-reported timings before they touch shared histograms.

    The envelope is produced inside the sandbox, where user code can forge it — an absurd
    but JSON-legal value like 1e300 would permanently poison the process-wide histogram
    sums for every tenant. Every phase is a sub-span of the run, so the backend-measured
    run duration (plus clock slack) is a hard ceiling; anything above it is clamped.
    """
    raw = envelope.get("timings") if isinstance(envelope, dict) else None
    if not isinstance(raw, dict):
        return {}
    return {
        _PHASE_BY_TIMING_KEY[key]: min(float(value), max_seconds)
        for key, value in raw.items()
        if key in _PHASE_BY_TIMING_KEY and isinstance(value, int | float) and math.isfinite(value) and value >= 0
    }


def _sanitized_delivery(envelope: Any) -> str:
    """Resolve the transport that served this run's rows, bounded to the known modes.

    A kernel run that read no ClickHouse-backed frame reports nothing, which is
    DELIVERY_NONE rather than a missing label: a run with no transport is a real bucket,
    and it must not be averaged in with the runs a transport comparison is about.
    """
    reported = envelope.get("delivery") if isinstance(envelope, dict) else None
    return reported if reported in DELIVERY_LABEL_VALUES else DELIVERY_NONE


def _capture_completed_event(
    run: NotebookNodeRun,
    outcome: str,
    duration: float,
    timings: dict[str, float],
    delivery: str,
) -> None:
    envelope = run.envelope if isinstance(run.envelope, dict) else {}
    properties: dict[str, Any] = {
        "notebook_short_id": run.notebook.short_id,
        "node_type": run.node_type,
        "outcome": outcome,
        "delivery": delivery,
        "duration_seconds": round(duration, 3),
        "row_count": envelope.get("row_count"),
        "has_error": bool(run.error),
        **{f"{phase}_seconds": round(value, 3) for phase, value in timings.items()},
    }
    try:
        user = run.user
    except ObjectDoesNotExist:
        # The FK is db_constraint=False/DO_NOTHING: a hard-deleted user leaves a dangling
        # id, and descriptor access raises instead of reading back as None.
        user = None
    # Always resolve the run's own team: with team=None, report_user_or_team_action
    # attributes $groups to the user's *currently active* project, which can differ
    # from the project the run belongs to (multi-project users, personal API keys).
    team = Team.objects.filter(pk=run.team_id).first()
    report_user_or_team_action(NODE_RUN_COMPLETED_EVENT, properties, user=user, team=team)